Add a role support to existing one-shot uploader for pulp_ansible about pulp_ansible HOT 5 CLOSED

commit b0038261704c50f5b44423b65de8621ee4e6b536
Author: Brian Bouterse <[email protected]>
Date:   Wed Oct 10 16:58:14 2018 -0400

    Adds a one-shot upload

    This one-shot upload will auto-discover any roles in the tarball
    uploaded, create necessary Roles, and RoleVersion objects, and then
    associate the RoleVersion objects with a new RepositoryVersion.

    https://pulp.plan.io/issues/4066
    closes #4066

diff --git a/README.rst b/README.rst
index 03fa20f..a7e7390 100644
--- a/README.rst
+++ b/README.rst
@@ -151,11 +151,35 @@ Look at the new Repository Version created
       "number": 1
   }

+Upload one or more Roles to Pulp (the easy way)
+-----------------------------------------------

-Upload a Role to Pulp
----------------------
+The upload API accepts a tarball which is opened up and any roles present will be imported and
+associated with the repository to create a new repository version.

-Download a role version.
+The created roles are assigned the following data:
+
+- The namespace is your username.
+- The role name is the role name of the directory in the uploaded tarball.
+- The version is an invented UUID due to version not being part of the Role metadata format. You can
+  assign versions later through the API.
+
+Here is a tarball with 6 roles in it.
+
+``curl -L https://github.com/pulp/ansible-pulp3/archive/master.tar.gz -o pulp.tar.gz``
+
+Upload it to Pulp and associate it with the repository:
+
+``http --form POST :8000/pulp_ansible/upload/ repository=$REPO_HREF [email protected] sha256=$(sha256sum pulp.tar.gz | awk '{ print $1 }')``
+
+
+Upload a Role to Pulp (the hard way)
+------------------------------------
+
+Uploading content this way lets you specify a namespace, name, and version which are automatically
+determined with the upload API above.
+
+To start "the hard way", download a role version.

 ``curl -L https://github.com/pulp/ansible-pulp3/archive/master.tar.gz -o pulp.tar.gz``

@@ -165,7 +189,7 @@ Create an Artifact by uploading the role version tarball to Pulp.


 Create a Role content unit
---------------------------
+^^^^^^^^^^^^^^^^^^^^^^^^^^

 Create an Ansible role in Pulp.

@@ -173,7 +197,7 @@ Create an Ansible role in Pulp.


 Create a ``role version`` from the Role and Artifact
------------------------------------------------------
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

 Create a content unit and point it to your Artifact and Role

@@ -181,13 +205,13 @@ Create a content unit and point it to your Artifact and Role


 Add content to repository ``foo``
----------------------------------
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

 ``$ http POST ':8000'$REPO_HREF'versions/' add_content_units:="[\"$CONTENT_HREF\"]"``


 Create a Publication
--------------------------------------------------
+--------------------

 ``$ http POST :8000/pulp/api/v3/ansible/publications/ repository=$REPO_HREF``

diff --git a/pulp_ansible/app/serializers.py b/pulp_ansible/app/serializers.py
index 7552559..addcae5 100644
--- a/pulp_ansible/app/serializers.py
+++ b/pulp_ansible/app/serializers.py
@@ -1,8 +1,10 @@
+from gettext import gettext as _
+
 from rest_framework import serializers

 from pulpcore.plugin.serializers import ContentSerializer, IdentityField, NestedIdentityField, \
     RelatedField, RemoteSerializer
-from pulpcore.plugin.models import Artifact
+from pulpcore.plugin.models import Artifact, Repository

 from .models import AnsibleRemote, AnsibleRole, AnsibleRoleVersion

@@ -61,3 +63,26 @@ class AnsibleRemoteSerializer(RemoteSerializer):
     class Meta:
         fields = RemoteSerializer.Meta.fields
         model = AnsibleRemote
+
+
+class OneShotUploadSerializer(serializers.Serializer):
+    """
+    A serializer for the One Shot Upload API.
+    """
+
+    repository = serializers.HyperlinkedRelatedField(
+        help_text=_('A URI of the repository.'),
+        required=True,
+        queryset=Repository.objects.all(),
+        view_name='repositories-detail',
+    )
+
+    file = serializers.FileField(
+        help_text=_("The collection file."),
+        required=True,
+    )
+
+    sha256 = serializers.CharField(
+        required=False,
+        default=None,
+    )
diff --git a/pulp_ansible/app/tasks/__init__.py b/pulp_ansible/app/tasks/__init__.py
index 67e6aac..925e12a 100644
--- a/pulp_ansible/app/tasks/__init__.py
+++ b/pulp_ansible/app/tasks/__init__.py
@@ -1,2 +1,3 @@
+from .upload import import_content_from_tarball  # noqa
 from .synchronizing import synchronize  # noqa
 from .publishing import publish  # noqa
diff --git a/pulp_ansible/app/tasks/upload.py b/pulp_ansible/app/tasks/upload.py
new file mode 100644
index 0000000..6349d69
--- /dev/null
+++ b/pulp_ansible/app/tasks/upload.py
@@ -0,0 +1,69 @@
+import os
+import re
+import tarfile
+import uuid
+
+from pulpcore.plugin.models import Artifact, ProgressBar, Repository, RepositoryVersion
+
+from pulp_ansible.app.models import AnsibleRole, AnsibleRoleVersion
+
+
+def import_content_from_tarball(namespace, artifact_pk=None, repository_pk=None):
+    """
+    Import Ansible content from a tarball saved as an Artifact.
+
+    The artifact is only a temporary storage area, and is deleted after being analyzed for more
+    content. Currently this task correctly handles: AnsibleRole and AnsibleRoleVersion content.
+
+    Args:
+        namespace (str): The namespace for any Ansible content to create
+        artifact_pk (int): The pk of the tarball Artifact to analyze and then delete
+        repository_pk (int): The repository that all created content should be associated with.
+    """
+    repository = Repository.objects.get(pk=repository_pk)
+    artifact = Artifact.objects.get(pk=artifact_pk)
+    role_paths = set()
+    with tarfile.open(str(artifact.file), "r") as tar:
+        artifact.delete()  # this artifact is only stored between the frontend and backend
+        for tarinfo in tar:
+            match = re.search('(.*)/(tasks|handlers|defaults|vars|files|templates|meta)/main.yml',
+                              tarinfo.path)
+            if match:
+                # This is a role asset
+                role_path = match.group(1)
+                role_paths.add(role_path)
+
+        tar.extractall()
+
+        role_version_pks = []
+        with ProgressBar(message='Importing Roles', total=len(role_paths)) as pb:
+            for role_path in role_paths:
+                match = re.search('(.*/)(.*)$', role_path)
+                role_name = match.group(2)
+                for tarinfo in tar:
+                    if tarinfo.path == role_path:
+                        # This is the role itself
+                        assert tarinfo.isdir()
+                        tarball_name = "{name}.tar.gz".format(name=role_name)
+                        with tarfile.open(tarball_name, "w:gz") as newtar:
+                            current_dir = os.getcwd()
+                            os.chdir(match.group(1))
+                            newtar.add(role_name)
+                            os.chdir(current_dir)
+                            full_path = os.path.abspath(tarball_name)
+                        new_artifact = Artifact.init_and_validate(full_path)
+                        new_artifact.save()
+                        role, created = AnsibleRole.objects.get_or_create(namespace=namespace,
+                                                                          name=role_name)
+                        version = uuid.uuid4()
+                        role_version = AnsibleRoleVersion(
+                            role=role,
+                            version=version
+                        )
+                        role_version.artifact = new_artifact
+                        role_version.save()
+                        role_version_pks.append(role_version.pk)
+                pb.increment()
+        with RepositoryVersion.create(repository) as new_version:
+            qs = AnsibleRoleVersion.objects.filter(pk__in=role_version_pks)
+            new_version.add_content(qs)
diff --git a/pulp_ansible/app/urls.py b/pulp_ansible/app/urls.py
index 4b7f872..efabbb3 100644
--- a/pulp_ansible/app/urls.py
+++ b/pulp_ansible/app/urls.py
@@ -3,10 +3,13 @@ from django.conf.urls import url
 from pulp_ansible.app.galaxy.views import (
     AnsibleGalaxyVersionView,
     AnsibleRoleList,
-    AnsibleRoleVersionList
+    AnsibleRoleVersionList,
 )
+from .viewsets import OneShotUploadView
+

 urlpatterns = [
+    url(r'pulp_ansible/upload/$', OneShotUploadView.as_view()),
     url(r'pulp_ansible/galaxy/(?P<path>.+)/api/$', AnsibleGalaxyVersionView.as_view()),
     url(r'pulp_ansible/galaxy/(?P<path>.+)/api/v1/roles/$', AnsibleRoleList.as_view()),
     url(r'pulp_ansible/galaxy/(?P<path>.+)/api/v1/roles/(?P<role_pk>[0-9a-f-]+)/versions/$',
diff --git a/pulp_ansible/app/viewsets.py b/pulp_ansible/app/viewsets.py
index 8106467..7b03209 100644
--- a/pulp_ansible/app/viewsets.py
+++ b/pulp_ansible/app/viewsets.py
@@ -1,7 +1,7 @@
 from django.db import transaction
 from drf_yasg.utils import swagger_auto_schema
 from rest_framework.decorators import detail_route
-from rest_framework import mixins, status
+from rest_framework import mixins, status, views
 from rest_framework.response import Response

 from pulpcore.plugin.models import Artifact, RepositoryVersion, Publication
@@ -22,7 +22,7 @@ from pulpcore.plugin.viewsets import (
 from . import tasks
 from .models import AnsibleRemote, AnsibleRole, AnsibleRoleVersion
 from .serializers import (AnsibleRemoteSerializer, AnsibleRoleSerializer,
-                          AnsibleRoleVersionSerializer)
+                          AnsibleRoleVersionSerializer, OneShotUploadSerializer)


 class AnsibleRoleFilter(BaseFilterSet):
@@ -188,3 +188,32 @@ class AnsiblePublicationsViewSet(NamedModelViewSet,
             }
         )
         return OperationPostponedResponse(result, request)
+
+
+class OneShotUploadView(views.APIView):
+    """
+    ViewSet for One Shot Upload API.
+    """
+
+    @transaction.atomic
+    def post(self, request):
+        """Upload an Ansible Role."""
+        serializer = OneShotUploadSerializer(
+            data=request.data, context={'request': request})
+        serializer.is_valid(raise_exception=True)
+        data = serializer.validated_data
+        expected_digests = {'sha256': data['sha256']}
+
+        artifact = Artifact.init_and_validate(request.data['file'],
+                                              expected_digests=expected_digests)
+        artifact.save()
+
+        repository = data['repository']
+        async_result = enqueue_with_reservation(
+            tasks.import_content_from_tarball, [repository],
+            kwargs={
+                'namespace': request.user.username,
+                'artifact_pk': artifact.pk,
+                'repository_pk': repository.pk
+            })
+        return OperationPostponedResponse(async_result, request)