Merge pull request #244 from mapswipe/dev

WIP: New Release 2.1.6

Author: Hagellach37

manager_dashboard/manager_dashboard/create.html

@@ -189,11 +189,15 @@ <h3>Project Type Specific Information</h3>
                               <label for="tileServerBuildArea">Tile Server Name</label>
                               <select name="tileServerBuildArea" id="tileServerBuildArea" onchange="displayTileServer(this, 'BuildArea', '')">
                                   <option value="bing">Bing</option>
-                                  <option value="digital_globe">Digital Globe</option>
+                                  <option value="mapbox">MapBox</option>
+                                  <option value="maxar_standard">Maxar Standard</option>
+                                  <option value="maxar_premium">Maxar Premium</option>
+                                  <option value="esri">Esri World Imagery</option>
+                                  <option value="esri_beta">Esri World Imagery (Clarity) Beta</option>
                                   <option value="sinergise">Sinergise</option>
                                   <option value="custom">Custom</option>
                               </select>
-                              <span>Select the tile server providing satellite imagery tiles for your project. Make sure you have permission. Google hasn't given permission yet and won't work until a proper API key is given.</span>
+                              <span>Select the tile server providing satellite imagery tiles for your project. Make sure you have permission.</span>
                           </li>
                           <li style="display:None" id="tileServerUrlFieldBuildArea">
                               <label for="tileServerUrlBuildArea">Custom Tile Server URL</label>
@@ -208,7 +212,7 @@ <h3>Project Type Specific Information</h3>
                           <li id="tileServerCreditsFieldBuildArea">
                               <label for="tileServerCreditsFieldBuildArea">Imagery Credits</label>
                               <input type="text" name="tileServerCreditsBuildArea" id="tileServerCreditsBuildArea" value="© 2019 Microsoft Corporation, Earthstar Geographics SIO">
-                              <span>Insert appropriate imagery credits. E.g. for Bing: "© 2019 Microsoft Corporation, Earthstar Geographics SIO"</span>
+                              <span>Insert appropriate imagery credits if you are using a custom tile server. For other tile server use the default credits.</span>
                           </li>
                           <li>
                               <input type="button" style="" id="submit" value="Submit" onClick="submitInfo()">
@@ -227,11 +231,15 @@ <h3>Project Type Specific Information</h3>
                               <label for="tileServerFootprint">Tile Server Name</label>
                               <select name="tileServerFootprint" id="tileServerFootprint" onchange="displayTileServer(this, 'Footprint', '')">
                                   <option value="bing">Bing</option>
-                                  <option value="digital_globe">Digital Globe</option>
+                                  <option value="mapbox">MapBox</option>
+                                  <option value="maxar_standard">Maxar Standard</option>
+                                  <option value="maxar_premium">Maxar Premium</option>
+                                  <option value="esri">Esri World Imagery</option>
+                                  <option value="esri_beta">Esri World Imagery (Clarity) Beta</option>
                                   <option value="sinergise">Sinergise</option>
                                   <option value="custom">Custom</option>
                               </select>
-                              <span>Select the tile server providing satellite imagery tiles for your project. Make sure you have permission. Google hasn't given permission yet and won't work until a proper API key is given.</span>
+                              <span>Select the tile server providing satellite imagery tiles for your project. Make sure you have permission.</span>
                           </li>
                           <li style="display:None" id="tileServerUrlFieldFootprint">
                               <label for="tileServerUrlFootprint">Custom Tile Server URL</label>
@@ -256,7 +264,7 @@ <h3>Project Type Specific Information</h3>
                           <li id="tileServerCreditsFieldFootprint">
                               <label for="tileServerCreditsFieldFootprint">Imagery Credits</label>
                               <input type="text" name="tileServerCreditsFootprint" id="tileServerCreditsFootprint" value="© 2019 Microsoft Corporation, Earthstar Geographics SIO">
-                              <span>Insert appropriate imagery credits. E.g. for Bing: "© 2019 Microsoft Corporation, Earthstar Geographics SIO"</span>
+                              <span>Insert appropriate imagery credits if you are using a custom tile server. For other tile server use the default credits.</span>
                           </li>
                           <li>
                               <input type="button" style="" id="submit" value="Submit" onClick="submitInfo()">
@@ -285,11 +293,15 @@ <h4>Tile Server A</h4>
                               <label for="tileServerChangeDetectionA">Tile Server Name</label>
                               <select name="tileServerChangeDetectionA" id="tileServerChangeDetectionA" onchange="displayTileServer(this, 'ChangeDetection', 'A')">
                                   <option value="bing">Bing</option>
-                                  <option value="digital_globe">Digital Globe</option>
+                                  <option value="mapbox">MapBox</option>
+                                  <option value="maxar_standard">Maxar Standard</option>
+                                  <option value="maxar_premium">Maxar Premium</option>
+                                  <option value="esri">Esri World Imagery</option>
+                                  <option value="esri_beta">Esri World Imagery (Clarity) Beta</option>
                                   <option value="sinergise">Sinergise</option>
                                   <option value="custom">Custom</option>
                               </select>
-                              <span>Select the tile server providing satellite imagery tiles for your project. Make sure you have permission. Google hasn't given permission yet and won't work until a proper API key is given.</span>
+                              <span>Select the tile server providing satellite imagery tiles for your project. Make sure you have permission.</span>
                           </li>
                           <li style="display:None" id="tileServerUrlFieldChangeDetectionA">
                               <label for="tileServerUrlChangeDetectionA">Custom Tile Server URL</label>
@@ -313,19 +325,23 @@ <h4>Tile Server A</h4>
                           </li>
                           <li id="tileServerCreditsFieldChangeDetectionA">
                               <label for="tileServerCreditsFieldChangeDetectionA">Imagery Credits</label>
-                              <input type="text" name="tileServerCreditsChangeDetectionA" id="tileServerCreditsChangeDetectionA">
-                              <span>Insert appropriate imagery credits. E.g. for Bing: "© 2019 Microsoft Corporation, Earthstar Geographics SIO"</span>
+                              <input type="text" name="tileServerCreditsChangeDetectionA" id="tileServerCreditsChangeDetectionA" value="© 2019 Microsoft Corporation, Earthstar Geographics SIO">
+                              <span>Insert appropriate imagery credits if you are using a custom tile server. For other tile server use the default credits.</span>
                           </li>
                           <h4>Tile Server B</h4>
                           <li>
                               <label for="tileServerChangeDetectionB">Tile Server Name</label>
                               <select name="tileServerChangeDetectionB" id="tileServerChangeDetectionB" onchange="displayTileServer(this, 'ChangeDetection', 'B')">
                                   <option value="bing">Bing</option>
-                                  <option value="digital_globe">Digital Globe</option>
+                                  <option value="mapbox">MapBox</option>
+                                  <option value="maxar_standard">Maxar Standard</option>
+                                  <option value="maxar_premium">Maxar Premium</option>
+                                  <option value="esri">Esri World Imagery</option>
+                                  <option value="esri_beta">Esri World Imagery (Clarity) Beta</option>
                                   <option value="sinergise">Sinergise</option>
                                   <option value="custom">Custom</option>
                               </select>
-                              <span>Select the tile server providing satellite imagery tiles for your project. Make sure you have permission. Google hasn't given permission yet and won't work until a proper API key is given.</span>
+                              <span>Select the tile server providing satellite imagery tiles for your project. Make sure you have permission.</span>
                           </li>
                           <li style="display:None" id="tileServerUrlFieldChangeDetectionB">
                               <label for="tileServerUrlChangeDetectionB">Custom Tile Server URL</label>
@@ -349,8 +365,8 @@ <h4>Tile Server B</h4>
                           </li>
                           <li id="tileServerCreditsFieldChangeDetectionB">
                               <label for="tileServerCreditsFieldChangeDetectionB">Imagery Credits</label>
-                              <input type="text" name="tileServerCreditsChangeDetectionB" id="tileServerCreditsChangeDetectionB">
-                              <span>Insert appropriate imagery credits. E.g. for Bing: "© 2019 Microsoft Corporation, Earthstar Geographics SIO"</span>
+                              <input type="text" name="tileServerCreditsChangeDetectionB" id="tileServerCreditsChangeDetectionB" value="© 2019 Microsoft Corporation, Earthstar Geographics SIO">
+                              <span>Insert appropriate imagery credits if you are using a custom tile server. For other tile server use the default credits.</span>
                           </li>
                           <li>
                               <input type="button" style="" id="submit" value="Submit" onClick="submitInfo()">

manager_dashboard/manager_dashboard/js/forms.js

@@ -30,6 +30,22 @@ function displayProjectTypeFormular(projectType) {
   }
 }
 
+function addTileServerCredits (tileServer, projectType, which) {
+    var credits = {
+        "bing": "© 2019 Microsoft Corporation, Earthstar Geographics SIO",
+        "maxar_premium": "© 2019 Maxar",
+        "maxar_standard": "© 2019 Maxar",
+        "esri": "© 2019 ESRI",
+        "esri_beta": "© 2019 ESRI",
+        "mapbox": "© 2019 MapBox",
+        "sinergise": "© 2019 Sinergise",
+        "custom": "Please add imagery credits here."
+    }
+
+    document.getElementById("tileServerCredits"+projectType+which).value = credits[tileServer]
+}
+
+
 function displayTileServer (t, projectType, which) {
     tileServer = t.value
     if (tileServer == "custom") {
@@ -42,6 +58,7 @@ function displayTileServer (t, projectType, which) {
         document.getElementById("tileServerUrlField"+projectType+which).style.display = "None";
         document.getElementById("tileServerLayerNameField"+projectType+which).style.display = "None";
     }
+    addTileServerCredits(tileServer, projectType, which)
 }
 
 function clear_all_fields() {

mapswipe_workers/.pre-commit-config.yaml

@@ -7,4 +7,4 @@ repos:
     rev: 3.7.9
     hooks:
       - id: flake8
-        args: ["--ignore=E501"]
+        args: ["--ignore=E501,W503"]

mapswipe_workers/mapswipe_workers/generate_stats/generate_stats.py

@@ -41,6 +41,8 @@ def generate_stats(project_id_list: list):
             logger.info(f"project {project_id} does not exist. skip this one.")
             continue
 
+        project_info = projects_df.loc[projects_df["project_id"] == project_id]
+
         logger.info(f"start generate stats for project: {project_id}")
         idx = projects_dynamic_df.index[
             projects_dynamic_df["project_id"] == project_id
@@ -49,7 +51,9 @@ def generate_stats(project_id_list: list):
             projects_dynamic_df.drop([idx[0]], inplace=True)
 
         # aggregate results and get per project statistics
-        project_stats_dict = project_stats.get_per_project_statistics(project_id)
+        project_stats_dict = project_stats.get_per_project_statistics(
+            project_id, project_info
+        )
         if project_stats_dict:
             projects_dynamic_df = projects_dynamic_df.append(
                 project_stats_dict, ignore_index=True
@@ -63,7 +67,9 @@ def generate_stats(project_id_list: list):
     if len(project_id_list) > 0:
         # merge static info and dynamic info and save
         projects_filename = f"{DATA_PATH}/api-data/projects/projects.csv"
-        projects_df = overall_stats.save_projects(projects_filename, projects_df, projects_dynamic_df)
+        projects_df = overall_stats.save_projects(
+            projects_filename, projects_df, projects_dynamic_df
+        )
 
         # generate overall stats for active, inactive, finished projects
         overall_stats_filename = f"{DATA_PATH}/api-data/stats.csv"

mapswipe_workers/mapswipe_workers/generate_stats/overall_stats.py

@@ -18,16 +18,20 @@ def get_overall_stats(projects_df: pd.DataFrame, filename: str) -> pd.DataFrame:
     filename: str
     """
 
-    overall_stats_df = projects_df.groupby(['status']).agg(
-        count_projects=pd.NamedAgg(column='project_id', aggfunc='count'),
-        area_sqkm=pd.NamedAgg(column='area_sqkm', aggfunc='sum'),
-        number_of_results=pd.NamedAgg(column='number_of_results', aggfunc='sum'),
-        number_of_results_progress=pd.NamedAgg(column='number_of_results_progress', aggfunc='sum'),
-        average_number_of_users_per_project=pd.NamedAgg(column='number_of_users', aggfunc='mean')
+    overall_stats_df = projects_df.groupby(["status"]).agg(
+        count_projects=pd.NamedAgg(column="project_id", aggfunc="count"),
+        area_sqkm=pd.NamedAgg(column="area_sqkm", aggfunc="sum"),
+        number_of_results=pd.NamedAgg(column="number_of_results", aggfunc="sum"),
+        number_of_results_progress=pd.NamedAgg(
+            column="number_of_results_progress", aggfunc="sum"
+        ),
+        average_number_of_users_per_project=pd.NamedAgg(
+            column="number_of_users", aggfunc="mean"
+        ),
     )
 
     overall_stats_df.to_csv(filename, index_label="status")
-    logger.info(f'saved overall stats to {filename}')
+    logger.info(f"saved overall stats to {filename}")
 
     return overall_stats_df
 
@@ -49,12 +53,17 @@ def get_project_static_info(filename: str) -> pd.DataFrame:
     # make sure to replace newline characters here
     sql_query = """
         COPY (
-            SELECT 
+            SELECT
                 project_id
                 ,regexp_replace(name, E'[\\n\\r]+', ' ', 'g' ) as name
                 ,regexp_replace(project_details, E'[\\n\\r]+', ' ', 'g' ) as project_details
                 ,regexp_replace(look_for, E'[\\n\\r]+', ' ', 'g' ) as look_for
                 ,project_type
+                -- add an array of the tile server names
+                ,CASE
+                  WHEN project_type_specifics->'tileServer'->'name' IS NOT NULL THEN Array[project_type_specifics->'tileServer'->>'name']
+                  ELSE Array[project_type_specifics->'tileServerA'->>'name', project_type_specifics->'tileServerB'->>'name']
+                END as tile_server_names
                 ,regexp_replace(status, E'[\\n\\r]+', ' ', 'g' ) as status
                 ,ST_Area(geom::geography)/1000000 as area_sqkm
                 ,ST_AsText(geom) as geom
@@ -101,7 +110,9 @@ def load_project_info_dynamic(filename: str) -> pd.DataFrame:
     return df
 
 
-def save_projects(filename: str, df: pd.DataFrame, df_dynamic: pd.DataFrame) -> pd.DataFrame:
+def save_projects(
+    filename: str, df: pd.DataFrame, df_dynamic: pd.DataFrame
+) -> pd.DataFrame:
     """
     The function merges the dataframes for static and dynamic project information
     and then save the result as csv file.
@@ -119,7 +130,7 @@ def save_projects(filename: str, df: pd.DataFrame, df_dynamic: pd.DataFrame) ->
     projects_df = df.merge(
         df_dynamic, left_on="project_id", right_on="project_id", how="left"
     )
-    projects_df.to_csv(filename, index_label="idx", line_terminator='\n')
+    projects_df.to_csv(filename, index_label="idx", line_terminator="\n")
     logger.info(f"saved projects: {filename}")
     geojson_functions.csv_to_geojson(filename, "geom")
     geojson_functions.csv_to_geojson(filename, "centroid")

mapswipe_workers/mapswipe_workers/generate_stats/project_stats.py

@@ -9,6 +9,17 @@
 from mapswipe_workers.generate_stats import project_stats_by_date
 
 
+def add_metadata_to_csv(filename: str):
+    """
+    Append a metadata line to the csv file about intended data usage.
+    """
+
+    with open(filename, "a") as fd:
+        fd.write("# This data can only be used for editing in OpenStreetMap.")
+
+    logger.info(f"added metadata to {filename}.")
+
+
 def write_sql_to_csv(filename: str, sql_query: sql.SQL):
     """
     Use the copy statement to write data from postgres to a csv file.
@@ -290,7 +301,7 @@ def get_agg_results_by_task_id(
     return agg_results_df
 
 
-def get_per_project_statistics(project_id: str) -> dict:
+def get_per_project_statistics(project_id: str, project_info: pd.Series) -> dict:
     """
     The function calculates all project related statistics.
     Always save results to csv file.
@@ -329,6 +340,10 @@ def get_per_project_statistics(project_id: str) -> dict:
         logger.info(f"saved agg results for {project_id}: {agg_results_filename}")
         geojson_functions.csv_to_geojson(agg_results_filename, "geom")
 
+        if any("maxar" in s for s in project_info["tile_server_names"]):
+            add_metadata_to_csv(agg_results_filename)
+            geojson_functions.add_metadata_to_geojson(agg_results_filename)
+
         project_stats_by_date_df = project_stats_by_date.get_project_history(
             results_df, groups_df
         )