vehicle-fleet-poc/config.yaml at main · livetocode/vehicle-fleet-poc · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
hub:
  type: nats
  enableProtoBuf: true
  protocols:
    nats:
      servers:
      - localhost:4222
    websockets:
      servers:
      - ws://localhost:4243

  # type: azureServiceBus
  # enableProtoBuf: true
  # connectionString: Endpoint=sb://foobar.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=<redacted>

  # type: amqp
  # enableProtoBuf: true
  # protocols:
  #   amqp:
  #     connectionString: amqp://localhost:5672
  #   websockets:
  #     server: ws://localhost:15670/ws

partitioning:
  timePartition:
    aggregationPeriodInMin: 10
    maxCapacity: 300000
    maxActivePartitions: 2
  dataPartition:
    # type: id
    # type: idGroup
    # groupSize: 10
    type: geohash
    hashLength: 5 # 4.9km x 4.9km

backpressure:
  enabled: true
  notificationThreshold: 1000
  notificationPeriodInMS: 100
  waitTimeoutInMS: 60000
  waitThreshold: 1000

chaosEngineering:
  enabled: false
  messageReadDelayInMS: 100
  messageWriteDelayInMS: 0

generator:
  logging:
    enabled: true
    level: debug
  instances: 1
  httpPort: 7710
  vehicleCount: 1500
  maxNumberOfEvents: 5000000
  messageChunkSize: 5000
  refreshIntervalInSecs: 5
  realtime: false
  sendFlush: true
  startDate: '2024-01-01T00:00:00-05:00'
  map:
    topLeftOrigin:
        # https://www.google.ca/maps/@45.6656598,-74.0651269,11.76z?entry=ttu
        lat: 45.6656598
        lon: -74.0651269
        alt: 11.76
    widthInKm: 40
    heightInKm: 20
  zoneSize:
    widthInKm: 10
    heightInKm: 10
  vehicleTypes:
    - Ambulance
    - Audi
    - Black_viper
    - Car
    - Mini_truck
    - Mini_van
    - Police
    - taxi
    - truck

collector:
  logging:
    enabled: true
    level: debug
  instances: 3
  httpPort: 7720
  geohashLength: 7 # 152.9m x 152.4m
  concurrentFlushes: true
  eventStore:
    type: noop # noop, memory, duckdb
  eventDispatcher:
    type: proxy
    dispatchers:
    - type: messageBus
    # - type: azureEventHub
    #   connectionString: EntityPath=move;Endpoint=sb://<REDACTED>.servicebus.windows.net/;<REDACTED>
    #   sendDelayInMS: 5000
    #   sendParallelism: 5

  output:
    overwriteExistingFiles: false
    # flatLayout should be false for the Rust finder using local files and Datafusion lib
    # flatLayout should be true when the finder relies on Azure Sql Server queries
    flatLayout: true
    formats:
    - parquet
    # - csv
    # - arrow
    storage:
      # type: noop
      # writeDelayInMS: 50

      # type: s3
      # bucketName: events
      # deleteBucketContentOnClear: true

      # type: azure-blob
      # accountName: <REDACTED>
      # containerName: events
      # connectionString: <REDACTED>

      type: file
      folder: ../../output/data
      isFake: false # if true, the collector won't actually write files to disk, but will simulate the write delay and log the file paths that would have been written. This is useful for testing the finder component without needing to generate large amounts of data or write to disk.
      writeDelayInMS: 1000 # simulate the time it takes to write a file to disk, only when isFake is true. This is for performance testing and debugging, to verify that the collector can cope with the message flow.

finder:
  logging:
    enabled: true
    level: debug
  runtime: nodejs # nodejs, rust
  instances: 3
  httpPort: 7730
  defaultTimeoutInMS: 30000
  dataFormat: parquet
  parallelSearch: true
  useChunking: false
  messageChunkSize: 5000
  dataSource:
    type: file # will use Collector.output.storage
    # type: azureSql
    # connection:
    #   server: '<REDACTED>-ondemand.sql.azuresynapse.net'
    #   database: 'vehicles'
    #   authentication:
    #       type: 'azure-active-directory-service-principal-secret'
    #       options:
    #           clientId: '<REDACTED>'
    #           clientSecret: '<REDACTED>'
    #           tenantId: '<REDACTED>'
    #   options:
    #       encrypt: true

viewer:
  logging:
    enabled: true
    level: debug