Scalability issues

datapackage/group.py

@@ -1,4 +1,5 @@
 from itertools import chain
+from datapackage import exceptions
 
 
 # Module API
@@ -52,3 +53,24 @@ def read(self, limit=None, **options):
             if count == limit:
                 break
         return rows
+
+    def check_relations(self):
+        # this function mimics the resource.check_relations but optimize it for groups
+        # it's also a prototype to proprose a faster check_relations process for all resources
+
+        # opti relations should ne loaded only once for the group
+        foreign_keys_values = self.__resources[0].get_foreign_keys_values()
+
+        # alternative to check_relations from tableschema-py
+        for resource in self.__resources:
+            try:
+                resource.check_relations(foreign_keys_values=foreign_keys_values)
+            except exceptions.DataPackageException as exception:
+                print('in %s: ' % resource.name)
+                if exception.multiple:
+                    for error in exception.errors:
+                        print(error)
+                else:
+                    print(exception)
+
+        return True

datapackage/resource.py

@@ -185,7 +185,7 @@ def iter(self, relations=False, **options):
 
         return self.__get_table().iter(relations=relations, **options)
 
-    def read(self, relations=False, **options):
+    def read(self, relations=False, foreign_keys_values=False, **options):
         """https://github.com/frictionlessdata/datapackage-py#resource
         """
 
@@ -195,17 +195,23 @@ def read(self, relations=False, **options):
             raise exceptions.DataPackageException(message)
 
         # Get relations
-        if relations:
+        if relations and not foreign_keys_values:
             relations = self.__get_relations()
 
-        return self.__get_table().read(relations=relations, **options)
+        return self.__get_table().read(relations=relations, foreign_keys_values=foreign_keys_values,
+                                       **options)
 
-    def check_relations(self):
+    def check_relations(self, foreign_keys_values=False):
         """https://github.com/frictionlessdata/datapackage-py#resource
         """
-        self.read(relations=True)
+        self.read(relations=True, foreign_keys_values=foreign_keys_values)
         return True
 
+    def drop_relations(self):
+        # storing relations datasets eats memory, we can need to garbage those
+        self.__relations = False
+        return self.__relations is False
+
     def raw_iter(self, stream=False):
         """https://github.com/frictionlessdata/datapackage-py#resource
         """
@@ -419,6 +425,10 @@ def __get_relations(self):
 
         return self.__relations
 
+    def get_foreign_keys_values(self):
+        # need to access it from groups for optimization
+        return self.__get_table().index_foreign_keys_values(self.__get_relations())
+
     # Deprecated
 
     @property