Filipe/substreams mappings (#1490)

* compile substreams mappings when available

* chore(dependencies): updated changesets for modified dependencies

* revert change to prettier

* run prettier

* it just works

* chore(dependencies): updated changesets for modified dependencies

* trigger ci

* make sure to uplaod mapping

* handle names of datasource gracefully

* ci: fix snapshot

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: Saihajpreet Singh <[email protected]>

Author: 3 people

.changeset/nervous-windows-serve.md

@@ -0,0 +1,5 @@
+---
+'@graphprotocol/graph-cli': minor
+---
+
+substreams based triggers support

packages/cli/src/compiler/index.ts

@@ -548,7 +548,7 @@ export default class Compiler {
                 );
             }
 
-            if (protocol.name == 'substreams') {
+            if (protocol.name == 'substreams' || protocol.name == 'substreams/triggers') {
               updatedDataSource = updatedDataSource
                 // Write data source ABIs to the output directory
                 .updateIn(['source', 'package'], (substreamsPackage: any) =>
@@ -569,6 +569,17 @@ export default class Compiler {
                   }),
                 );
 
+              if (updatedDataSource.getIn(['mapping', 'file'])) {
+                updatedDataSource = updatedDataSource.updateIn(
+                  ['mapping', 'file'],
+                  (mappingFile: string) =>
+                    path.relative(
+                      this.options.outputDir,
+                      path.resolve(this.sourceDir, mappingFile),
+                    ),
+                );
+              }
+
               return updatedDataSource;
             }
 
@@ -662,7 +673,7 @@ export default class Compiler {
         }
 
         // Upload all mappings
-        if (this.protocol.name === 'substreams') {
+        if (this.protocol.name === 'substreams' || this.protocol.name === 'substreams/triggers') {
           for (const [i, dataSource] of subgraph.get('dataSources').entries()) {
             updates.push({
               keyPath: ['dataSources', i, 'source', 'package', 'file'],
@@ -672,6 +683,17 @@ export default class Compiler {
                 spinner,
               ),
             });
+
+            if (dataSource.getIn(['mapping', 'file'])) {
+              updates.push({
+                keyPath: ['dataSources', i, 'mapping', 'file'],
+                value: await this._uploadFileToIPFS(
+                  dataSource.getIn(['mapping', 'file']),
+                  uploadedFiles,
+                  spinner,
+                ),
+              });
+            }
           }
         } else {
           for (const [i, dataSource] of subgraph.get('dataSources').entries()) {

packages/cli/src/protocols/index.ts

@@ -27,17 +27,24 @@ const protocolDebug = debug('graph-cli:protocol');
 
 export default class Protocol {
   static fromDataSources(dataSourcesAndTemplates: any) {
-    const firstDataSourceKind = dataSourcesAndTemplates[0].kind;
-    return new Protocol(firstDataSourceKind);
+    const firstDataSource = dataSourcesAndTemplates[0];
+    return new Protocol(firstDataSource);
   }
 
   name: ProtocolName;
 
   // TODO: should assert non null? see the constructor switch default case comment
   config!: ProtocolConfig;
 
-  constructor(name: ProtocolName) {
+  constructor(datasource: any) {
+    /**
+     * TODO: we should improve this `any` type, because some places
+     * we can initiate a Protocol with just a string (the name) and
+     * some other places use datasource object
+     */
+    const name = typeof datasource === 'string' ? datasource : datasource.kind;
     this.name = Protocol.normalizeName(name)!;
+    protocolDebug('Initializing protocol %s', this.name);
 
     switch (this.name) {
       case 'arweave':
@@ -54,6 +61,14 @@ export default class Protocol {
         break;
       case 'substreams':
         this.config = substreamsProtocol;
+
+        /**
+         * Substreams triggers are a special case of substreams data sources
+         * which have a mapping file and a handler.
+         */
+        if (datasource?.mapping?.file && datasource?.mapping.handler) {
+          this.name = 'substreams/triggers';
+        }
         break;
       default:
       // Do not throw when undefined, a better error message is printed after the constructor
@@ -126,7 +141,13 @@ export default class Protocol {
       ],
       substreams: ['mainnet'],
     }) as immutable.Map<
-      'arweave' | 'ethereum' | 'near' | 'cosmos' | 'substreams',
+      | 'arweave'
+      | 'ethereum'
+      | 'near'
+      | 'cosmos'
+      | 'substreams'
+      // this is temporary, until we have a better way to handle substreams triggers
+      | 'substreams/triggers',
       immutable.List<string>
     >;
   }
@@ -207,7 +228,13 @@ export default class Protocol {
   }
 }
 
-export type ProtocolName = 'arweave' | 'ethereum' | 'near' | 'cosmos' | 'substreams';
+export type ProtocolName =
+  | 'arweave'
+  | 'ethereum'
+  | 'near'
+  | 'cosmos'
+  | 'substreams'
+  | 'substreams/triggers';
 
 export interface ProtocolConfig {
   displayName: string;

packages/cli/src/protocols/substreams/manifest.graphql

@@ -32,14 +32,18 @@ type DataSource {
 
 type SubstreamsSource {
   package: SubstreamsPackage!
+  startBlock: BigInt
 }
 
 type SubstreamsPackage {
   moduleName: String!
   file: String!
+  params: String
 }
 
 type SubstreamMapping {
   apiVersion: String!
   kind: String!
+  file: File
+  handler: String
 }

packages/cli/src/subgraph.ts

@@ -53,7 +53,13 @@ export default class Subgraph {
     // Parse the default subgraph schema
     const schema = graphql.parse(
       await fs.readFile(
-        path.join(__dirname, 'protocols', protocol.name, `manifest.graphql`),
+        path.join(
+          __dirname,
+          'protocols',
+          // TODO: substreams/triggers is a special case, should be handled better
+          protocol.name === 'substreams/triggers' ? 'substreams' : protocol.name,
+          `manifest.graphql`,
+        ),
         'utf-8',
       ),
     );