Eurostat Test Data updation

scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/EurostatNUTS3_BirthDeathMigration.tmcf

@@ -5,45 +5,52 @@ variableMeasured: dcs:Count_Death
 observationAbout: C:EurostatNUTS3_BirthDeathMigration->geo
 observationDate: C:EurostatNUTS3_BirthDeathMigration->time
 value: C:EurostatNUTS3_BirthDeathMigration->Count_Death
+measurementMethod: dcs:EurostatRegionalStatistics
 
 Node: E:EurostatNUTS3_BirthDeathMigration->E1
 typeOf: dcs:StatVarObservation
-variableMeasured: dcs:Count_BirthEvent_AsAFractionOf_Count_Person
+variableMeasured: dcs:Count_BirthEvent_AsAFractionOfCount_Person
 observationAbout: C:EurostatNUTS3_BirthDeathMigration->geo
 observationDate: C:EurostatNUTS3_BirthDeathMigration->time
-value: C:EurostatNUTS3_BirthDeathMigration->Count_BirthEvent_AsAFractionOf_Count_Person
+value: C:EurostatNUTS3_BirthDeathMigration->Count_BirthEvent_AsAFractionOfCount_Person
+measurementMethod: dcs:EurostatRegionalStatistics
 
 Node: E:EurostatNUTS3_BirthDeathMigration->E2
 typeOf: dcs:StatVarObservation
-variableMeasured: dcs:Count_Death_AsAFractionOf_Count_Person
+variableMeasured: dcs:Count_Death_AsAFractionOfCount_Person
 observationAbout: C:EurostatNUTS3_BirthDeathMigration->geo
 observationDate: C:EurostatNUTS3_BirthDeathMigration->time
-value: C:EurostatNUTS3_BirthDeathMigration->Count_Death_AsAFractionOf_Count_Person
+value: C:EurostatNUTS3_BirthDeathMigration->Count_Death_AsAFractionOfCount_Person
+measurementMethod: dcs:EurostatRegionalStatistics
 
 Node: E:EurostatNUTS3_BirthDeathMigration->E3
 typeOf: dcs:StatVarObservation
 variableMeasured: dcs:IncrementalCount_Person
 observationAbout: C:EurostatNUTS3_BirthDeathMigration->geo
 observationDate: C:EurostatNUTS3_BirthDeathMigration->time
 value: C:EurostatNUTS3_BirthDeathMigration->IncrementalCount_Person
+measurementMethod: dcs:EurostatRegionalStatistics
 
 Node: E:EurostatNUTS3_BirthDeathMigration->E4
 typeOf: dcs:StatVarObservation
 variableMeasured: dcs:GrowthRate_Count_Person
 observationAbout: C:EurostatNUTS3_BirthDeathMigration->geo
 observationDate: C:EurostatNUTS3_BirthDeathMigration->time
 value: C:EurostatNUTS3_BirthDeathMigration->GrowthRate_Count_Person
+measurementMethod: dcs:EurostatRegionalStatistics
 
 Node: E:EurostatNUTS3_BirthDeathMigration->E5
 typeOf: dcs:StatVarObservation
 variableMeasured: dcs:Count_Person
 observationAbout: C:EurostatNUTS3_BirthDeathMigration->geo
 observationDate: C:EurostatNUTS3_BirthDeathMigration->time
 value: C:EurostatNUTS3_BirthDeathMigration->Count_Person
+measurementMethod: dcs:EurostatRegionalStatistics
 
 Node: E:EurostatNUTS3_BirthDeathMigration->E6
 typeOf: dcs:StatVarObservation
 variableMeasured: dcs:Count_BirthEvent
 observationAbout: C:EurostatNUTS3_BirthDeathMigration->geo
 observationDate: C:EurostatNUTS3_BirthDeathMigration->time
 value: C:EurostatNUTS3_BirthDeathMigration->Count_BirthEvent
+measurementMethod: dcs:EurostatRegionalStatistics

scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/EurostatNUTS3_BirthDeathMigration_test.py

@@ -0,0 +1,74 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+import os
+from import_data import preprocess_data, clean_data, download_data
+import pandas as pd
+import tempfile
+import sys
+
+MODULE_DIR = os.path.dirname(__file__)
+sys.path.insert(0, MODULE_DIR)
+
+TEST_DATASET_DIR = os.path.join(MODULE_DIR, "test_data", "sample_input")
+
+EXPECTED_FILES_DIR = os.path.join(MODULE_DIR, "test_data", "sample_output")
+
+
+class TestProcess(unittest.TestCase):
+    """
+    TestPreprocess is inherting unittest class
+    properties which further requried for unit testing.
+    The test will be conducted for EuroStat BMI Sample Datasets,
+    It will be generating CSV, MCF and TMCF files based on the sample input.
+    Comparing the data with the expected files.
+    """
+
+    def __init__(self, methodName: str = ...) -> None:
+        super().__init__(methodName)
+
+        CLEANED_CSV_FILE_PATH = os.path.join(EXPECTED_FILES_DIR,
+                                             "test_output.csv")
+        input_path = os.path.join(TEST_DATASET_DIR, 'sample_data.tsv')
+        input_df = pd.read_csv(input_path, sep='\s*\t\s*', engine='python')
+        input_df = input_df.rename(
+            columns={'freq,indic_de,geo\TIME_PERIOD': 'indic_de,geo\\time'})
+        input_df['indic_de,geo\\time'] = input_df[
+            'indic_de,geo\\time'].str.slice(2)
+        preprocess_df = preprocess_data(input_df)
+        clean_data(preprocess_df, CLEANED_CSV_FILE_PATH)
+
+        with open(CLEANED_CSV_FILE_PATH, encoding="utf-8-sig") as csv_file:
+            self.actual_csv_data = csv_file.read()
+
+    def test_create_csv(self):
+        """
+        This method is required to test between output generated
+        preprocess script and excepted output files like CSV
+        """
+        expected_csv_file_path = os.path.join(
+            EXPECTED_FILES_DIR, "EurostatNUTS3_BirthDeathMigration.csv")
+
+        expected_csv_data = ""
+        with open(expected_csv_file_path,
+                  encoding="utf-8") as expected_csv_file:
+            expected_csv_data = expected_csv_file.read()
+
+        self.assertEqual(expected_csv_data.strip(),
+                         self.actual_csv_data.strip())
+
+
+if __name__ == "__main__":
+    unittest.main()

scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/README.md

@@ -67,8 +67,12 @@ python3 import_data.py
 
 ### Testing Procedure
 
+How to run: Generate sample input and output CSV files.
+
+
 To test import procedure, run the following command:
 
 ```
-python3 test_import.py
+python3 EurostatNUTS3_BirthDeathMigration_test.py
+
 ```

scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/import_data.py

@@ -13,13 +13,24 @@
 # limitations under the License.
 
 import pandas as pd
+from six.moves import urllib
+import sys
+
+sys.path.insert(1, '../../../../util')
+from alpha2_to_dcid import COUNTRY_MAP
+from nuts_codes_names import NUTS1_CODES_NAMES
 
 
 def download_data(download_link):
     """Downloads raw data from Eurostat website and stores it in instance
     data frame.
     """
-    raw_df = pd.read_table(download_link)
+    urllib.request.urlretrieve(download_link, "demo_r_gind3.tsv.gz")
+    raw_df = pd.read_table("demo_r_gind3.tsv.gz")
+    raw_df = raw_df.rename(columns=({
+        'freq,indic_de,geo\TIME_PERIOD': 'indic_de,geo\\time'
+    }))
+    raw_df['indic_de,geo\\time'] = raw_df['indic_de,geo\\time'].str.slice(2)
     return raw_df
 
 
@@ -74,7 +85,7 @@ def preprocess_data(raw_df):
 
     # Append extra space for all cells in value column that do not come with a note, so that we can split them without error.
     preprocessed_df.value = preprocessed_df.value.str.replace(
-        "([0-9:])$", lambda m: m.group(0) + ' ')
+        "([0-9:])$", lambda m: m.group(0) + ' ', regex=True)
 
     first_column_list = preprocessed_df.columns[0].rsplit(sep=",", maxsplit=1)
 
@@ -126,8 +137,10 @@ def clean_data(preprocessed_df, output_path):
 
     # replace colon with NaN.
     clean_df = clean_df.replace(':', '')
+    clean_df['geo'] = clean_df['geo'].apply(lambda geo: f'nuts/{geo}' if any(
+        geo.isdigit() for geo in geo) or ('nuts/' + geo in NUTS1_CODES_NAMES)
+                                            else COUNTRY_MAP.get(geo, f'{geo}'))
 
-    clean_df['geo'] = 'dcid:nuts/' + clean_df['geo']
     # trim the space in the time column i.e. '2020 ' -> '2020'
     clean_df['time'] = clean_df['time'].astype('int32')
     original_names = [
@@ -136,8 +149,8 @@ def clean_data(preprocessed_df, output_path):
     ]
     new_names = [
         'geo', 'time', 'Count_Death',
-        'Count_BirthEvent_AsAFractionOf_Count_Person',
-        'Count_Death_AsAFractionOf_Count_Person', 'IncrementalCount_Person',
+        'Count_BirthEvent_AsAFractionOfCount_Person',
+        'Count_Death_AsAFractionOfCount_Person', 'IncrementalCount_Person',
         'GrowthRate_Count_Person', 'Count_Person', 'Count_BirthEvent'
     ]
     clean_df = clean_df[original_names]
@@ -151,7 +164,7 @@ def clean_data(preprocessed_df, output_path):
 
 
 if __name__ == '__main__':
-    download_link = "https://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?file=data/demo_r_gind3.tsv.gz"
+    download_link = "https://ec.europa.eu/eurostat/api/dissemination/sdmx/2.1/data/demo_r_gind3/?format=TSV&compressed=true"
     output_path = 'EurostatNUTS3_BirthDeathMigration.csv'
     raw_df = download_data(download_link)
     preprocessed_df = preprocess_data(raw_df)

scripts/eurostat/regional_statistics_by_nuts/birth_death_migration/manifest.json

@@ -0,0 +1,17 @@
+{
+    "import_specifications": [
+        {
+            "import_name": "EurostatData_Demographic_Balance_Crude_Rates",
+            "curator_emails": ["[email protected]"],
+            "provenance_url":"https://ec.europa.eu/eurostat/api/dissemination/sdmx/2.1/data/demo_r_gind3/?format=TSV&compressed=true",
+            "provenance_description": "Regional Statistics by NUTS Classification.",
+            "scripts": ["import_data.py"],
+            "import_inputs": [{
+                "template_mcf": "EurostatNUTS3_BirthDeathMigration.tmcf",
+                "cleaned_csv": "EurostatNUTS3_BirthDeathMigration.csv"
+            }],
+        "cron_schedule": "0 10 * * 2"
+        }
+    ]
+}
+