-
Notifications
You must be signed in to change notification settings - Fork 1
Feat/reward token #10
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
99bc59e
c83bd12
561b502
ffc570a
7f55dc6
fbedd4a
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -34,6 +34,10 @@ class WriterManager | |
| private bool is_anonymous; | ||
| private bool is_upload_automatically; | ||
| public static int amount_compressed_file = 0; | ||
| public static int disk_event_counter = 0; | ||
| public static int file_event_counter = 0; | ||
| public static TimeSpan active_session = TimeSpan.FromSeconds(0); | ||
| public static TimeSpan trace_duration = TimeSpan.FromSeconds(0); | ||
|
|
||
| public WriterManager(string dirpath, bool is_anonymous, bool upload,ObjectStorageHandler obj) | ||
| { | ||
|
|
@@ -58,6 +62,8 @@ public WriterManager(string dirpath, bool is_anonymous, bool upload,ObjectStorag | |
| process_snap_filepath = GenerateFilePath("process"); | ||
| fs_snap_filepath = GenerateFilePath("filesystem_snapshot"); | ||
| this.is_anonymous = is_anonymous; | ||
|
|
||
| StartEventRateDetector(); | ||
| } | ||
|
|
||
| public void InitiateDirectory() | ||
|
|
@@ -95,6 +101,32 @@ public void InitiateDirectory() | |
| Console.WriteLine("File output: {0}", this.dir_path); | ||
| } | ||
|
|
||
| private void StartEventRateDetector() | ||
| { | ||
| Debug.WriteLine("Starting event rate detector thread..."); | ||
| Thread eventRateThread = new(EventRateDetector) | ||
| { | ||
| IsBackground = true | ||
| }; | ||
| eventRateThread.Start(); | ||
| } | ||
|
|
||
| private void EventRateDetector() | ||
| { | ||
| while (true) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
| { | ||
| int initial_count = disk_event_counter; | ||
| Thread.Sleep(1000); | ||
| int final_count = disk_event_counter; | ||
| int events_in_interval = final_count - initial_count; | ||
| disk_event_counter = 0; | ||
|
Comment on lines
+118
to
+122
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This block of code for measuring the event rate is not thread-safe. |
||
| //Debug.WriteLine($"Rate: {events_in_interval}"); | ||
| if (events_in_interval > 100) { | ||
| active_session += TimeSpan.FromSeconds(1); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private static string EscapeCsvField(string field) | ||
| { | ||
| if (string.IsNullOrEmpty(field)) | ||
|
|
@@ -145,6 +177,8 @@ public void Write(FilesystemTrace data) | |
| { | ||
| return; | ||
| } | ||
| file_event_counter += 1; | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
| //event_counter += 1; | ||
| fs_sb.Append(data.FormatAsCsv(is_anonymous)); | ||
| if (IsTimeToFlush(fs_sb)) | ||
| { | ||
|
|
@@ -165,7 +199,7 @@ public void Write(DiskTrace data) | |
| { | ||
| return; | ||
| } | ||
|
|
||
| disk_event_counter += 1; | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
| ds_sb.Append(data.FormatAsCsv()); | ||
|
|
||
| if (IsTimeToFlush(ds_sb)) | ||
|
|
@@ -187,7 +221,7 @@ public void Write(NetworkTrace data) | |
| { | ||
| return; | ||
| } | ||
|
|
||
| //event_counter += 1; | ||
| nw_sb.Append(data.FormatAsCsv()); | ||
|
|
||
| if (IsTimeToFlush(nw_sb)) | ||
|
|
@@ -279,6 +313,8 @@ public void DirectWrite(string file_out_path ,string input) | |
| writer.Write(input); | ||
| } | ||
|
|
||
| amount_compressed_file++; | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
|
|
||
| if (is_upload_automatically) | ||
| { | ||
| obj_storage.QueueFile(out_path); | ||
|
|
@@ -361,10 +397,6 @@ public void CompressRun() | |
| Directory.Delete(dir_path, true); | ||
| } | ||
|
|
||
| public void FlushSnapper() | ||
| { | ||
| FlushWrite(fs_snap_sb, fs_snap_filepath, "filesystem_snapshot"); | ||
| } | ||
|
|
||
| public async Task CompressAllAsync() | ||
| { | ||
|
|
@@ -380,6 +412,7 @@ public async Task CompressAllAsync() | |
| WriteStatus(); | ||
|
|
||
| await obj_storage.ClearQueue(); | ||
| ConfigClasses.SaveTracemetaConfiguration(active_session + trace_duration, file_event_counter); | ||
|
|
||
| CompressRun(); | ||
| } | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -17,6 +17,8 @@ internal class ObjectStorageHandler | |
| private R2Client r2Client; | ||
| private ConcurrentQueue<string> uploadQueue = new(); | ||
| public static int UploadedFiles = 0; | ||
| private TimeSpan LastActiveHours = TimeSpan.FromSeconds(0); | ||
| public static int LastFileEvent; | ||
|
|
||
|
|
||
| public ObjectStorageHandler() | ||
|
|
@@ -26,9 +28,16 @@ public ObjectStorageHandler() | |
|
|
||
| public async Task UploadFile(string filepath) | ||
| { | ||
| //double deltaSeconds = WriterManager.active_session.TotalSeconds - LastActiveHours.TotalSeconds; | ||
| //int deltaFileEvent = WriterManager.file_event_counter - LastFileEvent; | ||
| //Debug.WriteLine($"Uploading file with delta seconds: {deltaSeconds}, delta file events: {deltaFileEvent}"); | ||
|
|
||
| FileInfo fi = new FileInfo(filepath); | ||
| await r2Client.PutObject(fi); | ||
| File.Delete(filepath); | ||
|
|
||
| LastFileEvent = WriterManager.file_event_counter; | ||
| LastActiveHours = WriterManager.active_session; | ||
| } | ||
|
|
||
| public void QueueFile(string filepath) | ||
|
|
@@ -54,7 +63,7 @@ public async Task ClearQueue() | |
| catch (Exception ex) | ||
| { | ||
| Debug.WriteLine($"Error uploading {filepath}: {ex}"); | ||
| QueueFile(filepath); | ||
| //QueueFile(filepath); | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The logic to re-queue a file for upload upon failure has been commented out. This means that if an upload fails due to a transient issue (e.g., a temporary network error), the file will be lost instead of being retried. This could lead to data loss. If this change is intentional, it should be documented. Otherwise, the retry logic should be restored, perhaps with a limited number of retries and a backoff strategy. |
||
| } | ||
| } | ||
| } | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,83 @@ | ||
| using IOTracesCORE.cloudstorage; | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Diagnostics; | ||
| using System.Linq; | ||
| using System.Security.Cryptography; | ||
| using System.Text; | ||
| using System.Text.Json; | ||
| using System.Threading.Tasks; | ||
|
|
||
| namespace IOTracesCORE.utils | ||
| { | ||
| internal class ConfigClasses | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
| { | ||
| public class PersistedConfig | ||
| { | ||
| public string OutputPath { get; set; } = ""; | ||
| public bool Anonymous { get; set; } | ||
| public bool UploadEnabled { get; set; } | ||
| } | ||
|
|
||
| public class TraceMetadata | ||
| { | ||
| public long TraceDuration { get; set; } | ||
| public long FileEventsCount { get; set; } | ||
| } | ||
|
|
||
| public static string AppConfigPath => | ||
| Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), | ||
| "IOTracer", "config.json.enc"); | ||
|
|
||
| public static string ConfigTraceMetadataPath => | ||
| Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), | ||
| "IOTracer", "trace_metadata.json.enc"); | ||
|
|
||
| public static void SaveTracemetaConfiguration(TimeSpan trace_active, long file_events) | ||
| { | ||
| try | ||
| { | ||
| Debug.WriteLine($"Saving a session of {trace_active.TotalSeconds} seconds"); | ||
| var cfg = new TraceMetadata | ||
| { | ||
| TraceDuration = (long)trace_active.TotalSeconds, | ||
| FileEventsCount = file_events | ||
| }; | ||
|
|
||
| string json = JsonSerializer.Serialize(cfg, new JsonSerializerOptions { WriteIndented = true }); | ||
| byte[] encrypted = ProtectedData.Protect( | ||
| Encoding.UTF8.GetBytes(json), null, DataProtectionScope.CurrentUser); | ||
| Directory.CreateDirectory(Path.GetDirectoryName(ConfigTraceMetadataPath)!); | ||
| File.WriteAllBytes(ConfigTraceMetadataPath, encrypted); | ||
| } | ||
| catch (Exception ex) { Debug.WriteLine("Save config failed: " + ex.Message); } | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This catch (Exception ex) { Debug.WriteLine("Save config failed: " + ex.ToString()); } |
||
| } | ||
|
|
||
| public static void LoadTracemetaConfiguration() | ||
| { | ||
| try | ||
| { | ||
| Debug.Write($"Receiving config from: {ConfigTraceMetadataPath}"); | ||
| if (!File.Exists(ConfigTraceMetadataPath)) return; | ||
|
|
||
| byte[] encrypted = File.ReadAllBytes(ConfigTraceMetadataPath); | ||
| byte[] decrypted = ProtectedData.Unprotect(encrypted, null, DataProtectionScope.CurrentUser); | ||
| string json = Encoding.UTF8.GetString(decrypted); | ||
| var cfg = JsonSerializer.Deserialize<TraceMetadata>(json); | ||
|
|
||
| if (cfg == null) return; | ||
|
|
||
| WriterManager.trace_duration = TimeSpan.FromSeconds(cfg.TraceDuration); | ||
| WriterManager.file_event_counter += (int)cfg.FileEventsCount; | ||
| ObjectStorageHandler.LastFileEvent = (int)cfg.FileEventsCount; | ||
|
Comment on lines
+71
to
+72
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Casting |
||
| Debug.WriteLine($"Loaded a session of {cfg.TraceDuration} seconds"); | ||
| Debug.WriteLine($"Loaded {cfg.FileEventsCount} file events"); | ||
| } | ||
| catch (Exception ex) | ||
| { | ||
| Debug.WriteLine("Load config failed: " + ex.Message); | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
|
|
||
| } | ||
| } | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The local variable names
Total_current_sessionandTotal_trace_durationdo not follow standard C# naming conventions, which recommendcamelCasefor local variables (e.g.,totalCurrentSession). Adhering to conventions improves code readability and maintainability. Please update these variables and their subsequent usages.