[Fixes #12368] Introduce geonode-importer in geonode-core (#12570)

* [Fixes #12368] Test Include Importer in GeoNode
* [Fixes #12368] Removed unused function/deadcode/files
* [Fixes #12368] Fix dataset assign for sld and xml
* [Fixes #12368] add pre-validation step
* [Fixes #12368] fix serializers for overwrite workflow
* [Fixes #12368] fix replace
* [Fixes #12368] Fix sld serializer

Author: mattiagiupponi

.circleci/config.yml

@@ -110,7 +110,7 @@ workflows:
           codecov_name: main_tests
           load_docker_cache: false
           save_docker_cache: false
-          test_suite: ./test.sh $(python -c "import sys;from geonode import settings;sys.stdout.write('\'' '\''.join([a+'\''.tests'\'' for a in settings.GEONODE_APPS if '\''security'\'' not in a and '\''geoserver'\'' not in a]))") geonode.thumbs.tests geonode.people.tests geonode.people.socialaccount.providers.geonode_openid_connect.tests
+          test_suite: ./test.sh $(python -c "import sys;from geonode import settings;sys.stdout.write('\'' '\''.join([a+'\''.tests'\'' for a in settings.GEONODE_APPS if '\''security'\'' not in a and '\''geoserver'\'' not in a and '\''upload'\'' not in a]))") geonode.thumbs.tests geonode.people.tests geonode.people.socialaccount.providers.geonode_openid_connect.tests
       - build:
           name: geonode_test_security
           codecov_name: security_tests
@@ -128,14 +128,19 @@ workflows:
           codecov_name: api
           load_docker_cache: false
           save_docker_cache: false
-          test_suite: ./test.sh geonode.api.tests geonode.base.api.tests geonode.layers.api.tests geonode.maps.api.tests geonode.documents.api.tests geonode.geoapps.api.tests geonode.upload.api.tests
+          test_suite: ./test.sh geonode.api.tests geonode.base.api.tests geonode.layers.api.tests geonode.maps.api.tests geonode.documents.api.tests geonode.geoapps.api.tests
       - build:
           name: geonode_test_csw
           codecov_name: csw
           load_docker_cache: false
           save_docker_cache: false
           test_suite: ./test.sh geonode.tests.csw geonode.catalogue.backends.tests
-
+      - build:
+          name: geonode_upload
+          codecov_name: importer
+          load_docker_cache: false
+          save_docker_cache: false
+          test_suite: ./test.sh geonode.upload          
       # TODO
       # - build:
       #     name: geonode_test_integration_upload

.env_test

@@ -39,7 +39,7 @@ GEONODE_DB_CONN_TOUT=5
 DEFAULT_BACKEND_DATASTORE=datastore
 BROKER_URL=amqp://guest:guest@rabbitmq:5672/
 CELERY_BEAT_SCHEDULER=celery.beat:PersistentScheduler
-ASYNC_SIGNALS=True
+ASYNC_SIGNALS=False
 
 SITEURL=http://localhost:8000/
 
@@ -64,7 +64,7 @@ NGINX_BASE_URL=http://localhost
 # port where the server can be reached on HTTPS
 HTTP_HOST=localhost
 HTTPS_HOST=
-
+POSTGRESQL_MAX_CONNECTIONS=100
 HTTP_PORT=8000
 HTTPS_PORT=443
 
@@ -237,4 +237,4 @@ DEFAULT_MAX_PARALLEL_UPLOADS_PER_USER=100
 MICROSOFT_TENANT_ID=
 AZURE_CLIENT_ID=
 AZURE_SECRET_KEY=
-AZURE_KEY=
+AZURE_KEY=

docker-compose-test.yml

@@ -80,15 +80,15 @@ services:
       retries: 5
       start_period: 30s
 
-  # Gets and installs letsencrypt certificates
-  letsencrypt:
-    image: geonode/letsencrypt:2.6.0-latest
-    container_name: letsencrypt4${COMPOSE_PROJECT_NAME}
-    env_file:
-      - .env_test
-    volumes:
-      - nginx-certificates:/geonode-certificates
-    restart: unless-stopped
+  # # Gets and installs letsencrypt certificates
+  # letsencrypt:
+  #   image: geonode/letsencrypt:2.6.0-latest
+  #   container_name: letsencrypt4${COMPOSE_PROJECT_NAME}
+  #   env_file:
+  #     - .env_test
+  #   volumes:
+  #     - nginx-certificates:/geonode-certificates
+  #   restart: unless-stopped
 
   # Geoserver backend
   geoserver:

geonode/__init__.py

@@ -28,7 +28,7 @@ def get_version():
     return geonode.version.get_version(__version__)
 
 
-def main(global_settings, **settings):
+def main(_, **settings):
     from django.core.wsgi import get_wsgi_application
 
     os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings.get("django_settings"))

geonode/assets/tests.py

@@ -226,7 +226,26 @@ def test_download_file(self):
         u, _ = get_user_model().objects.get_or_create(username="admin")
         self.assertTrue(self.client.login(username="admin", password="admin"), "Login failed")
 
-        asset = self._setup_test(u)
+        asset_handler = asset_handler_registry.get_default_handler()
+        asset = asset_handler.create(
+            title="Test Asset",
+            description="Description of test asset",
+            type="NeverMind",
+            owner=u,
+            files=[ONE_JSON],
+            clone_files=True,
+        )
+        asset.save()
+        self.assertIsInstance(asset, LocalAsset)
+
+        reloaded = LocalAsset.objects.get(pk=asset.pk)
+
+        # put two more files in the asset dir
+        asset_dir = os.path.dirname(reloaded.location[0])
+        sub_dir = os.path.join(asset_dir, "subdir")
+        os.mkdir(sub_dir)
+        shutil.copy(TWO_JSON, asset_dir)
+        shutil.copy(THREE_JSON, sub_dir)
 
         for path, key in ((None, "one"), ("one.json", "one"), ("two.json", "two"), ("subdir/three.json", "three")):
             # args = [asset.pk, path] if path else [asset.pk]

geonode/base/__init__.py

@@ -72,9 +72,3 @@ def register_event(request, event_type, resource):
         raise ValueError(f"Invalid resource: {resource}")
     if request and hasattr(request, "register_event"):
         request.register_event(event_type, resource_type, resource_name, resource_id)
-
-
-def register_proxy_event(request):
-    """
-    Process request to geoserver proxy. Extract layer and ows type
-    """

geonode/base/api/tests.py

@@ -2408,17 +2408,9 @@ def test_resource_service_copy_with_perms_doc(self):
 
     @override_settings(CELERY_TASK_ALWAYS_EAGER=True)
     def test_resource_service_copy_with_perms_map(self):
-        files = os.path.join(gisdata.GOOD_DATA, "vector/single_point.shp")
-        files_as_dict, _ = get_files(files)
-        resource = Document.objects.create(
-            owner=get_user_model().objects.get(username="admin"),
-            alternate="geonode:test_copy",
-            resource_type="map",
-            uuid=str(uuid4()),
-        )
-        _, _ = create_asset_and_link(
-            resource, get_user_model().objects.get(username="admin"), list(files_as_dict.values())
-        )
+
+        resource = create_single_map(name="test_copy")
+
         self._assertCloningWithPerms(resource)
 
     def _assertCloningWithPerms(self, resource):
@@ -2430,7 +2422,7 @@ def _assertCloningWithPerms(self, resource):
         resource.set_permissions(_perms)
         copy_url = reverse("importer_resource_copy", kwargs={"pk": resource.pk})
         response = self.client.put(copy_url, data={"title": "cloned_resource"})
-        self.assertIn(response.status_code, [403, 404])
+        self.assertIn(response.status_code, [302, 403, 404])
         # set perms to enable user clone resource
         # bobby can copy the resource since he has all the perms needed
         _perms = {

geonode/base/auth.py

@@ -47,23 +47,6 @@ def extract_user_from_headers(request):
     return user
 
 
-def extract_headers(request):
-    """
-    Extracts headers from the Django request object
-    :param request: The current django.http.HttpRequest object
-    :return: a dictionary with OAuthLib needed headers
-    """
-    headers = request.META.copy()
-    if "wsgi.input" in headers:
-        del headers["wsgi.input"]
-    if "wsgi.errors" in headers:
-        del headers["wsgi.errors"]
-    if "HTTP_AUTHORIZATION" in headers:
-        headers["Authorization"] = headers["HTTP_AUTHORIZATION"]
-
-    return headers
-
-
 def make_token_expiration(seconds=86400):
     _expire_seconds = getattr(settings, "ACCESS_TOKEN_EXPIRE_SECONDS", seconds)
     _expire_time = datetime.datetime.now(timezone.get_current_timezone())

geonode/base/bbox_utils.py

@@ -62,13 +62,6 @@ def as_polygon(self):
         return DjangoPolygon.from_bbox((self.xmin, self.ymin, self.xmax, self.ymax))
 
 
-def normalize_x_value(value):
-    """
-    Normalise x-axis value/longtitude to fall within [-180, 180]
-    """
-    return ((value + 180) % 360) - 180
-
-
 def polygon_from_bbox(bbox, srid=4326):
     """
     Constructs a Polygon object with srid from a provided bbox.

geonode/base/forms.py

@@ -29,7 +29,6 @@
 from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.contrib.auth.models import Group
-from django.core import validators
 from django.db.models import Prefetch, Q
 from django.forms import models
 from django.forms.fields import ChoiceField, MultipleChoiceField
@@ -659,22 +658,6 @@ class Meta:
         )
 
 
-class ValuesListField(forms.Field):
-    def to_python(self, value):
-        if value in validators.EMPTY_VALUES:
-            return []
-
-        value = [item.strip() for item in value.split(",") if item.strip()]
-
-        return value
-
-    def clean(self, value):
-        value = self.to_python(value)
-        self.validate(value)
-        self.run_validators(value)
-        return value
-
-
 class BatchEditForm(forms.Form):
     LANGUAGES = (("", "--------"),) + ALL_LANGUAGES
     group = forms.ModelChoiceField(label=_("Group"), queryset=Group.objects.all(), required=False)

geonode/base/models.py

@@ -617,15 +617,6 @@ def cleanup_uploaded_files(resource_id):
             filename = f"{_resource.get_real_instance().resource_type}-{_resource.get_real_instance().uuid}"
             remove_thumbs(filename)
 
-            # Remove the uploaded sessions, if any
-            if "geonode.upload" in settings.INSTALLED_APPS:
-                from geonode.upload.models import Upload
-
-                # Need to call delete one by one in order to invoke the
-                #  'delete' overridden method
-                for upload in Upload.objects.filter(resource_id=_resource.get_real_instance().id):
-                    upload.delete()
-
 
 class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase):
     """

geonode/catalogue/views.py

@@ -201,16 +201,6 @@ def fst(value):
     return result
 
 
-# from a resource object, build the corresponding metadata dict
-# the aim is to handle the output format (csv, html or pdf) the same structure
-def build_md_dict(resource):
-    md_dict = {
-        "r_uuid": {"label": "uuid", "value": resource.uuid},
-        "r_title": {"label": "titre", "value": resource.title},
-    }
-    return md_dict
-
-
 def get_keywords(resource):
     content = " "
     cursor = connection.cursor()

geonode/client/conf.py

@@ -38,14 +38,6 @@ def load_path_attr(path):
     return attr
 
 
-def is_installed(package):
-    try:
-        __import__(package)
-        return True
-    except ImportError:
-        return False
-
-
 class GeoNodeClientAppConf(AppConf):
     LAYER_PREVIEW_LIBRARY = "geonode"
     HOOKSET = "geonode.client.hooksets.BaseHookSet"

geonode/decorators.py

@@ -153,29 +153,6 @@ def view_or_apiauth(view, request, test_func, *args, **kwargs):
     return response
 
 
-def has_perm_or_basicauth(perm, realm=""):
-    """
-    This is similar to the above decorator 'logged_in_or_basicauth'
-    except that it requires the logged in user to have a specific
-    permission.
-
-    Use:
-
-    @logged_in_or_basicauth('asforums.view_forumcollection')
-    def your_view:
-        ...
-
-    """
-
-    def view_decorator(func):
-        def wrapper(request, *args, **kwargs):
-            return view_or_basicauth(func, request, lambda u: u.has_perm(perm), realm, *args, **kwargs)
-
-        return wrapper
-
-    return view_decorator
-
-
 def superuser_only(function):
     """
     Limit view to superusers only.
@@ -289,16 +266,6 @@ def wrapper(request, *args, **kwargs):
     return view_decorator
 
 
-def logged_in_or_apiauth():
-    def view_decorator(func):
-        def wrapper(request, *args, **kwargs):
-            return view_or_apiauth(func, request, lambda u: u.is_authenticated, *args, **kwargs)
-
-        return wrapper
-
-    return view_decorator
-
-
 def superuser_or_apiauth():
     def view_decorator(func):
         def wrapper(request, *args, **kwargs):
@@ -307,11 +274,3 @@ def wrapper(request, *args, **kwargs):
         return wrapper
 
     return view_decorator
-
-
-def dump_func_name(func):
-    def echo_func(*func_args, **func_kwargs):
-        logger.debug(f"Start func: {func.__name__}")
-        return func(*func_args, **func_kwargs)
-
-    return echo_func

geonode/documents/forms.py

@@ -110,12 +110,6 @@ class Meta(ResourceBaseForm.Meta):
         )
 
 
-class DocumentDescriptionForm(forms.Form):
-    title = forms.CharField(max_length=300)
-    abstract = forms.CharField(max_length=2000, widget=forms.Textarea, required=False)
-    keywords = forms.CharField(max_length=500, required=False)
-
-
 class DocumentCreateForm(TranslationModelForm):
     """
     The document upload form.