From 8c30ed6d090be2ad9e03bfc570ee940795488dc9 Mon Sep 17 00:00:00 2001
From: german77 <juangerman-13@hotmail.com>
Date: Sat, 1 May 2021 17:02:35 -0500
Subject: [PATCH] hid: Improve hardware accuracy of gestures
---
.../hle/service/hid/controllers/gesture.cpp | 347 +++++++++++++++---
.../hle/service/hid/controllers/gesture.h | 69 +++-
2 files changed, 340 insertions(+), 76 deletions(-)
diff --git a/src/core/hle/service/hid/controllers/gesture.cpp b/src/core/hle/service/hid/controllers/gesture.cpp
index bb77d89593..9e5df3bb79 100644
--- a/src/core/hle/service/hid/controllers/gesture.cpp
+++ b/src/core/hle/service/hid/controllers/gesture.cpp
@@ -1,10 +1,9 @@
-// Copyright 2018 yuzu emulator team
+// Copyright 2021 yuzu Emulator Project
// Licensed under GPLv2 or any later version
// Refer to the license.txt file included.
-#include <cstring>
-#include "common/common_types.h"
#include "common/logging/log.h"
+#include "common/math_util.h"
#include "common/settings.h"
#include "core/core_timing.h"
#include "core/frontend/emu_window.h"
@@ -12,10 +11,19 @@
namespace Service::HID {
constexpr std::size_t SHARED_MEMORY_OFFSET = 0x3BA00;
-constexpr f32 angle_threshold = 0.08f;
-constexpr f32 pinch_threshold = 100.0f;
-Controller_Gesture::Controller_Gesture(Core::System& system_) : ControllerBase{system_} {}
+// HW is around 700, value is set to 400 to make it easier to trigger with mouse
+constexpr f32 swipe_threshold = 400.0f; // Threshold in pixels/s
+constexpr f32 angle_threshold = 0.015f; // Threshold in radians
+constexpr f32 pinch_threshold = 0.5f; // Threshold in pixels
+constexpr f32 press_delay = 0.5f; // Time in seconds
+constexpr f32 double_tap_delay = 0.35f; // Time in seconds
+
+constexpr f32 Square(s32 num) {
+ return static_cast<f32>(num * num);
+}
+
+Controller_Gesture::Controller_Gesture(Core::System& system) : ControllerBase(system) {}
Controller_Gesture::~Controller_Gesture() = default;
void Controller_Gesture::OnInit() {
@@ -24,6 +32,8 @@ void Controller_Gesture::OnInit() {
keyboard_finger_id[id] = MAX_POINTS;
udp_finger_id[id] = MAX_POINTS;
}
+ shared_memory.header.entry_count = 0;
+ force_update = true;
}
void Controller_Gesture::OnRelease() {}
@@ -38,17 +48,23 @@ void Controller_Gesture::OnUpdate(const Core::Timing::CoreTiming& core_timing, u
shared_memory.header.last_entry_index = 0;
return;
}
- shared_memory.header.entry_count = 16;
- const auto& last_entry = shared_memory.gesture_states[shared_memory.header.last_entry_index];
- shared_memory.header.last_entry_index = (shared_memory.header.last_entry_index + 1) % 17;
- auto& cur_entry = shared_memory.gesture_states[shared_memory.header.last_entry_index];
+ ReadTouchInput();
- cur_entry.sampling_number = last_entry.sampling_number + 1;
- cur_entry.sampling_number2 = cur_entry.sampling_number;
+ GestureProperties gesture = GetGestureProperties();
+ f32 time_difference = static_cast<f32>(shared_memory.header.timestamp - last_update_timestamp) /
+ (1000 * 1000 * 1000);
- // TODO(german77): Implement all gesture types
+ // Only update if necesary
+ if (!ShouldUpdateGesture(gesture, time_difference)) {
+ return;
+ }
+ last_update_timestamp = shared_memory.header.timestamp;
+ UpdateGestureSharedMemory(data, size, gesture, time_difference);
+}
+
+void Controller_Gesture::ReadTouchInput() {
const Input::TouchStatus& mouse_status = touch_mouse_device->GetStatus();
const Input::TouchStatus& udp_status = touch_udp_device->GetStatus();
for (std::size_t id = 0; id < mouse_status.size(); ++id) {
@@ -63,50 +79,71 @@ void Controller_Gesture::OnUpdate(const Core::Timing::CoreTiming& core_timing, u
UpdateTouchInputEvent(keyboard_status[id], keyboard_finger_id[id]);
}
}
+}
+bool Controller_Gesture::ShouldUpdateGesture(const GestureProperties& gesture,
+ f32 time_difference) {
+ const auto& last_entry = shared_memory.gesture_states[shared_memory.header.last_entry_index];
+ if (force_update) {
+ force_update = false;
+ return true;
+ }
+
+ // Update if coordinates change
+ for (size_t id = 0; id < MAX_POINTS; id++) {
+ if (gesture.points[id].x != last_gesture.points[id].x ||
+ gesture.points[id].y != last_gesture.points[id].y) {
+ return true;
+ }
+ }
+
+ // Update on press and hold event after 0.5 seconds
+ if (last_entry.type == TouchType::Touch && last_entry.point_count == 1 &&
+ time_difference > press_delay) {
+ return enable_press_and_tap;
+ }
+
+ return false;
+}
+
+void Controller_Gesture::UpdateGestureSharedMemory(u8* data, std::size_t size,
+ GestureProperties& gesture,
+ f32 time_difference) {
TouchType type = TouchType::Idle;
Attribute attributes{};
- GestureProperties gesture = GetGestureProperties();
- if (last_gesture.active_points != gesture.active_points) {
- ++last_gesture.detection_count;
+
+ const auto& last_entry = shared_memory.gesture_states[shared_memory.header.last_entry_index];
+ shared_memory.header.last_entry_index = (shared_memory.header.last_entry_index + 1) % 17;
+ auto& cur_entry = shared_memory.gesture_states[shared_memory.header.last_entry_index];
+
+ if (shared_memory.header.entry_count < 16) {
+ shared_memory.header.entry_count++;
}
+
+ cur_entry.sampling_number = last_entry.sampling_number + 1;
+ cur_entry.sampling_number2 = cur_entry.sampling_number;
+
+ // Reset values to default
+ cur_entry.delta_x = 0;
+ cur_entry.delta_y = 0;
+ cur_entry.vel_x = 0;
+ cur_entry.vel_y = 0;
+ cur_entry.direction = Direction::None;
+ cur_entry.rotation_angle = 0;
+ cur_entry.scale = 0;
+
if (gesture.active_points > 0) {
if (last_gesture.active_points == 0) {
- attributes.is_new_touch.Assign(true);
- last_gesture.average_distance = gesture.average_distance;
- last_gesture.angle = gesture.angle;
+ NewGesture(gesture, type, attributes);
+ } else {
+ UpdateExistingGesture(gesture, type, time_difference);
}
-
- type = TouchType::Touch;
- if (gesture.mid_point.x != last_entry.x || gesture.mid_point.y != last_entry.y) {
- type = TouchType::Pan;
- }
- if (std::abs(gesture.average_distance - last_gesture.average_distance) > pinch_threshold) {
- type = TouchType::Pinch;
- }
- if (std::abs(gesture.angle - last_gesture.angle) > angle_threshold) {
- type = TouchType::Rotate;
- }
-
- cur_entry.delta_x = gesture.mid_point.x - last_entry.x;
- cur_entry.delta_y = gesture.mid_point.y - last_entry.y;
- // TODO: Find how velocities are calculated
- cur_entry.vel_x = static_cast<float>(cur_entry.delta_x) * 150.1f;
- cur_entry.vel_y = static_cast<float>(cur_entry.delta_y) * 150.1f;
-
- // Slowdown the rate of change for less flapping
- last_gesture.average_distance =
- (last_gesture.average_distance * 0.9f) + (gesture.average_distance * 0.1f);
- last_gesture.angle = (last_gesture.angle * 0.9f) + (gesture.angle * 0.1f);
-
} else {
- cur_entry.delta_x = 0;
- cur_entry.delta_y = 0;
- cur_entry.vel_x = 0;
- cur_entry.vel_y = 0;
+ EndGesture(gesture, last_gesture, type, attributes, time_difference);
}
- last_gesture.active_points = gesture.active_points;
- cur_entry.detection_count = last_gesture.detection_count;
+
+ // Apply attributes
+ cur_entry.detection_count = gesture.detection_count;
cur_entry.type = type;
cur_entry.attributes = attributes;
cur_entry.x = gesture.mid_point.x;
@@ -116,12 +153,190 @@ void Controller_Gesture::OnUpdate(const Core::Timing::CoreTiming& core_timing, u
cur_entry.points[id].x = gesture.points[id].x;
cur_entry.points[id].y = gesture.points[id].y;
}
- cur_entry.rotation_angle = 0;
- cur_entry.scale = 0;
+ last_gesture = gesture;
std::memcpy(data + SHARED_MEMORY_OFFSET, &shared_memory, sizeof(SharedMemory));
}
+void Controller_Gesture::NewGesture(GestureProperties& gesture, TouchType& type,
+ Attribute& attributes) {
+ const auto& last_entry =
+ shared_memory.gesture_states[(shared_memory.header.last_entry_index + 16) % 17];
+ gesture.detection_count++;
+ type = TouchType::Touch;
+
+ // New touch after cancel is not considered new
+ if (last_entry.type != TouchType::Cancel) {
+ attributes.is_new_touch.Assign(1);
+ enable_press_and_tap = true;
+ }
+}
+
+void Controller_Gesture::UpdateExistingGesture(GestureProperties& gesture, TouchType& type,
+ f32 time_difference) {
+ const auto& last_entry =
+ shared_memory.gesture_states[(shared_memory.header.last_entry_index + 16) % 17];
+
+ // Promote to pan type if touch moved
+ for (size_t id = 0; id < MAX_POINTS; id++) {
+ if (gesture.points[id].x != last_gesture.points[id].x ||
+ gesture.points[id].y != last_gesture.points[id].y) {
+ type = TouchType::Pan;
+ break;
+ }
+ }
+
+ // Number of fingers changed cancel the last event and clear data
+ if (gesture.active_points != last_gesture.active_points) {
+ type = TouchType::Cancel;
+ enable_press_and_tap = false;
+ gesture.active_points = 0;
+ gesture.mid_point = {};
+ for (size_t id = 0; id < MAX_POINTS; id++) {
+ gesture.points[id].x = 0;
+ gesture.points[id].y = 0;
+ }
+ return;
+ }
+
+ // Calculate extra parameters of panning
+ if (type == TouchType::Pan) {
+ UpdatePanEvent(gesture, last_gesture, type, time_difference);
+ return;
+ }
+
+ // Promote to press type
+ if (last_entry.type == TouchType::Touch) {
+ type = TouchType::Press;
+ }
+}
+
+void Controller_Gesture::EndGesture(GestureProperties& gesture, GestureProperties& last_gesture,
+ TouchType& type, Attribute& attributes, f32 time_difference) {
+ const auto& last_entry =
+ shared_memory.gesture_states[(shared_memory.header.last_entry_index + 16) % 17];
+ if (last_gesture.active_points != 0) {
+ switch (last_entry.type) {
+ case TouchType::Touch:
+ if (enable_press_and_tap) {
+ SetTapEvent(gesture, last_gesture, type, attributes);
+ return;
+ }
+ type = TouchType::Cancel;
+ force_update = true;
+ break;
+ case TouchType::Press:
+ case TouchType::Tap:
+ case TouchType::Swipe:
+ case TouchType::Pinch:
+ case TouchType::Rotate:
+ type = TouchType::Complete;
+ force_update = true;
+ break;
+ case TouchType::Pan:
+ EndPanEvent(gesture, last_gesture, type, time_difference);
+ break;
+ default:
+ break;
+ }
+ return;
+ }
+ if (last_entry.type == TouchType::Complete || last_entry.type == TouchType::Cancel) {
+ gesture.detection_count++;
+ }
+}
+
+void Controller_Gesture::SetTapEvent(GestureProperties& gesture, GestureProperties& last_gesture,
+ TouchType& type, Attribute& attributes) {
+ type = TouchType::Tap;
+ gesture = last_gesture;
+ force_update = true;
+ f32 tap_time_difference =
+ static_cast<f32>(last_update_timestamp - last_tap_timestamp) / (1000 * 1000 * 1000);
+ last_tap_timestamp = last_update_timestamp;
+ if (tap_time_difference < double_tap_delay) {
+ attributes.is_double_tap.Assign(1);
+ }
+}
+
+void Controller_Gesture::UpdatePanEvent(GestureProperties& gesture, GestureProperties& last_gesture,
+ TouchType& type, f32 time_difference) {
+ auto& cur_entry = shared_memory.gesture_states[shared_memory.header.last_entry_index];
+ const auto& last_entry =
+ shared_memory.gesture_states[(shared_memory.header.last_entry_index + 16) % 17];
+ cur_entry.delta_x = gesture.mid_point.x - last_entry.x;
+ cur_entry.delta_y = gesture.mid_point.y - last_entry.y;
+
+ cur_entry.vel_x = static_cast<f32>(cur_entry.delta_x) / time_difference;
+ cur_entry.vel_y = static_cast<f32>(cur_entry.delta_y) / time_difference;
+ last_pan_time_difference = time_difference;
+
+ // Promote to pinch type
+ if (std::abs(gesture.average_distance - last_gesture.average_distance) > pinch_threshold) {
+ type = TouchType::Pinch;
+ cur_entry.scale = gesture.average_distance / last_gesture.average_distance;
+ }
+
+ const f32 angle_between_two_lines = std::atan((gesture.angle - last_gesture.angle) /
+ (1 + (gesture.angle * last_gesture.angle)));
+ // Promote to rotate type
+ if (std::abs(angle_between_two_lines) > angle_threshold) {
+ type = TouchType::Rotate;
+ cur_entry.scale = 0;
+ cur_entry.rotation_angle = angle_between_two_lines * 180.0f / Common::PI;
+ }
+}
+
+void Controller_Gesture::EndPanEvent(GestureProperties& gesture, GestureProperties& last_gesture,
+ TouchType& type, f32 time_difference) {
+ auto& cur_entry = shared_memory.gesture_states[shared_memory.header.last_entry_index];
+ const auto& last_entry =
+ shared_memory.gesture_states[(shared_memory.header.last_entry_index + 16) % 17];
+ cur_entry.vel_x =
+ static_cast<f32>(last_entry.delta_x) / (last_pan_time_difference + time_difference);
+ cur_entry.vel_y =
+ static_cast<f32>(last_entry.delta_y) / (last_pan_time_difference + time_difference);
+ const f32 curr_vel =
+ std::sqrt((cur_entry.vel_x * cur_entry.vel_x) + (cur_entry.vel_y * cur_entry.vel_y));
+
+ // Set swipe event with parameters
+ if (curr_vel > swipe_threshold) {
+ SetSwipeEvent(gesture, last_gesture, type);
+ return;
+ }
+
+ // End panning without swipe
+ type = TouchType::Complete;
+ cur_entry.vel_x = 0;
+ cur_entry.vel_y = 0;
+ force_update = true;
+}
+
+void Controller_Gesture::SetSwipeEvent(GestureProperties& gesture, GestureProperties& last_gesture,
+ TouchType& type) {
+ auto& cur_entry = shared_memory.gesture_states[shared_memory.header.last_entry_index];
+ const auto& last_entry =
+ shared_memory.gesture_states[(shared_memory.header.last_entry_index + 16) % 17];
+ type = TouchType::Swipe;
+ gesture = last_gesture;
+ force_update = true;
+ cur_entry.delta_x = last_entry.delta_x;
+ cur_entry.delta_y = last_entry.delta_y;
+ if (std::abs(cur_entry.delta_x) > std::abs(cur_entry.delta_y)) {
+ if (cur_entry.delta_x > 0) {
+ cur_entry.direction = Direction::Right;
+ return;
+ }
+ cur_entry.direction = Direction::Left;
+ return;
+ }
+ if (cur_entry.delta_y > 0) {
+ cur_entry.direction = Direction::Down;
+ return;
+ }
+ cur_entry.direction = Direction::Up;
+}
+
void Controller_Gesture::OnLoadInputDevices() {
touch_mouse_device = Input::CreateDevice<Input::TouchDevice>("engine:emu_window");
touch_udp_device = Input::CreateDevice<Input::TouchDevice>("engine:cemuhookudp");
@@ -183,23 +398,33 @@ Controller_Gesture::GestureProperties Controller_Gesture::GetGestureProperties()
for (size_t id = 0; id < gesture.active_points; ++id) {
gesture.points[id].x =
- static_cast<int>(active_fingers[id].x * Layout::ScreenUndocked::Width);
+ static_cast<s32>(active_fingers[id].x * Layout::ScreenUndocked::Width);
gesture.points[id].y =
- static_cast<int>(active_fingers[id].y * Layout::ScreenUndocked::Height);
- gesture.mid_point.x += static_cast<int>(gesture.points[id].x / gesture.active_points);
- gesture.mid_point.y += static_cast<int>(gesture.points[id].y / gesture.active_points);
+ static_cast<s32>(active_fingers[id].y * Layout::ScreenUndocked::Height);
+
+ // Hack: There is no touch in docked but games still allow it
+ if (Settings::values.use_docked_mode.GetValue()) {
+ gesture.points[id].x =
+ static_cast<s32>(active_fingers[id].x * Layout::ScreenDocked::Width);
+ gesture.points[id].y =
+ static_cast<s32>(active_fingers[id].y * Layout::ScreenDocked::Height);
+ }
+
+ gesture.mid_point.x += static_cast<s32>(gesture.points[id].x / gesture.active_points);
+ gesture.mid_point.y += static_cast<s32>(gesture.points[id].y / gesture.active_points);
}
for (size_t id = 0; id < gesture.active_points; ++id) {
- const double distance =
- std::pow(static_cast<float>(gesture.mid_point.x - gesture.points[id].x), 2) +
- std::pow(static_cast<float>(gesture.mid_point.y - gesture.points[id].y), 2);
- gesture.average_distance +=
- static_cast<float>(distance) / static_cast<float>(gesture.active_points);
+ const f32 distance = std::sqrt(Square(gesture.mid_point.x - gesture.points[id].x) +
+ Square(gesture.mid_point.y - gesture.points[id].y));
+ gesture.average_distance += distance / static_cast<f32>(gesture.active_points);
}
- gesture.angle = std::atan2(static_cast<float>(gesture.mid_point.y - gesture.points[0].y),
- static_cast<float>(gesture.mid_point.x - gesture.points[0].x));
+ gesture.angle = std::atan2(static_cast<f32>(gesture.mid_point.y - gesture.points[0].y),
+ static_cast<f32>(gesture.mid_point.x - gesture.points[0].x));
+
+ gesture.detection_count = last_gesture.detection_count;
+
return gesture;
}
diff --git a/src/core/hle/service/hid/controllers/gesture.h b/src/core/hle/service/hid/controllers/gesture.h
index 7c357b9772..18110a6adb 100644
--- a/src/core/hle/service/hid/controllers/gesture.h
+++ b/src/core/hle/service/hid/controllers/gesture.h
@@ -1,4 +1,4 @@
-// Copyright 2018 yuzu emulator team
+// Copyright 2021 yuzu Emulator Project
// Licensed under GPLv2 or any later version
// Refer to the license.txt file included.
@@ -7,7 +7,6 @@
#include <array>
#include "common/bit_field.h"
#include "common/common_types.h"
-#include "common/swap.h"
#include "core/frontend/input.h"
#include "core/hle/service/hid/controllers/controller_base.h"
@@ -35,10 +34,10 @@ private:
enum class TouchType : u32 {
Idle, // Nothing touching the screen
- Complete, // Unknown. End of touch?
- Cancel, // Never triggered
- Touch, // Pressing without movement
- Press, // Never triggered
+ Complete, // Set at the end of a touch event
+ Cancel, // Set when the number of fingers change
+ Touch, // A finger just touched the screen
+ Press, // Set if last type is touch and the finger hasn't moved
Tap, // Fast press then release
Pan, // All points moving together across the screen
Swipe, // Fast press movement and release of a single point
@@ -58,8 +57,8 @@ private:
union {
u32_le raw{};
- BitField<0, 1, u32> is_new_touch;
- BitField<1, 1, u32> is_double_tap;
+ BitField<4, 1, u32> is_new_touch;
+ BitField<8, 1, u32> is_double_tap;
};
};
static_assert(sizeof(Attribute) == 4, "Attribute is an invalid size");
@@ -73,10 +72,9 @@ private:
struct GestureState {
s64_le sampling_number;
s64_le sampling_number2;
-
s64_le detection_count;
TouchType type;
- Direction dir;
+ Direction direction;
s32_le x;
s32_le y;
s32_le delta_x;
@@ -84,8 +82,8 @@ private:
f32 vel_x;
f32 vel_y;
Attribute attributes;
- u32 scale;
- u32 rotation_angle;
+ f32 scale;
+ f32 rotation_angle;
s32_le point_count;
std::array<Points, 4> points;
};
@@ -109,10 +107,46 @@ private:
Points mid_point{};
s64_le detection_count{};
u64_le delta_time{};
- float average_distance{};
- float angle{};
+ f32 average_distance{};
+ f32 angle{};
};
+ // Reads input from all available input engines
+ void ReadTouchInput();
+
+ // Returns true if gesture state needs to be updated
+ bool ShouldUpdateGesture(const GestureProperties& gesture, f32 time_difference);
+
+ // Updates the shared memory to the next state
+ void UpdateGestureSharedMemory(u8* data, std::size_t size, GestureProperties& gesture,
+ f32 time_difference);
+
+ // Initializes new gesture
+ void NewGesture(GestureProperties& gesture, TouchType& type, Attribute& attributes);
+
+ // Updates existing gesture state
+ void UpdateExistingGesture(GestureProperties& gesture, TouchType& type, f32 time_difference);
+
+ // Terminates exiting gesture
+ void EndGesture(GestureProperties& gesture, GestureProperties& last_gesture, TouchType& type,
+ Attribute& attributes, f32 time_difference);
+
+ // Set current event to a tap event
+ void SetTapEvent(GestureProperties& gesture, GestureProperties& last_gesture, TouchType& type,
+ Attribute& attributes);
+
+ // Calculates and set the extra parameters related to a pan event
+ void UpdatePanEvent(GestureProperties& gesture, GestureProperties& last_gesture,
+ TouchType& type, f32 time_difference);
+
+ // Terminates the pan event
+ void EndPanEvent(GestureProperties& gesture, GestureProperties& last_gesture, TouchType& type,
+ f32 time_difference);
+
+ // Set current event to a swipe event
+ void SetSwipeEvent(GestureProperties& gesture, GestureProperties& last_gesture,
+ TouchType& type);
+
// Returns an unused finger id, if there is no fingers avaliable MAX_FINGERS will be returned
std::optional<size_t> GetUnusedFingerID() const;
@@ -134,6 +168,11 @@ private:
std::array<size_t, MAX_FINGERS> keyboard_finger_id;
std::array<size_t, MAX_FINGERS> udp_finger_id;
std::array<Finger, MAX_POINTS> fingers;
- GestureProperties last_gesture;
+ GestureProperties last_gesture{};
+ s64_le last_update_timestamp{};
+ s64_le last_tap_timestamp{};
+ f32 last_pan_time_difference{};
+ bool force_update{false};
+ bool enable_press_and_tap{false};
};
} // namespace Service::HID