feat(projectHistoryLogs): log new submissions (#5416)

### 📣 Summary
Create logs when new submissions are added to projects.


### 👷 Description for instance maintainers
Allow null user_uids in AuditLogs so we can log anonymous submissions.


### 💭 Notes
We previously had no need for null users in audit logs because the
actions we logged were all restricted to authenticated users, but since
we allow anonymous submissions, we needed a way to log those.



### 👀 Preview steps

Feature/no-change template:
1. ℹ️ have an account and a project. Make sure the account username is
not `admin` (see [this notion
task](https://www.notion.so/kobotoolbox/Anonymous-submissions-dont-work-if-user-named-admin-owns-asset-1767e515f65480608dfcee76ba9b3710?pvs=4))
2. Deploy the project
3. Add a submission to the project
4. Go to `api/v2/asset/<asset-uid>/history`
5. 🟢 There should be a new project history log with
`action='add-submission'` and all the usual metadata, plus
```
"submission": {
    "submitted_by": "user1"
}
```
6. Enable submissions without username/email to the project
7. To make sure you're submitting anonymously, copy and paste the enketo
link into a new private tab and add a new submission
8. 🟢 Reload the endpoint. There should be a new audit log with
`action='add-submission'`
a. The user should be
`http://kf.kobo.local:8080/api/v2/users/AnonymousUser/`
  b. The user_uid will be the uid of the anonymous user in the database
  c. The username should be `AnonymousUser`
d. The metadata should contain `{"submission": {"submitted_by":
"AnonymousUser"}` in addition to the usual

kobo/apps/audit_log/models.py

@@ -38,6 +38,7 @@
 from kpi.fields.kpi_uid import UUID_LENGTH
 from kpi.models import Asset, ImportTask
 from kpi.utils.log import logging
+from kpi.utils.object_permission import get_database_user
 
 ANONYMOUS_USER_PERMISSION_ACTIONS = {
     # key: (permission, granting?), value: ph log action
@@ -396,6 +397,8 @@ def create_from_request(cls, request: WSGIRequest):
             'submission-validation-statuses': cls._create_from_submission_request,
             'submission-validation-status': cls._create_from_submission_request,
             'assetsnapshot-submission-alias': cls._create_from_submission_request,
+            'submissions': cls._create_from_submission_request,
+            'submissions-list': cls._create_from_submission_request,
         }
         url_name = request.resolver_match.url_name
         method = url_name_to_action.get(url_name, None)
@@ -608,7 +611,7 @@ def _create_from_submission_request(cls, request):
         instances: dict[int:SubmissionUpdate] = getattr(request, 'instances', {})
         logs = []
         url_name = request.resolver_match.url_name
-
+        user = get_database_user(request.user)
         for instance in instances.values():
             if instance.action == 'add':
                 action = AuditAction.ADD_SUBMISSION
@@ -628,10 +631,10 @@ def _create_from_submission_request(cls, request):
 
             logs.append(
                 ProjectHistoryLog(
-                    user=request.user,
+                    user=user,
                     object_id=request.asset.id,
                     action=action,
-                    user_uid=request.user.extra_details.uid,
+                    user_uid=user.extra_details.uid,
                     metadata=metadata,
                 )
             )

kobo/apps/audit_log/signals.py

@@ -77,8 +77,13 @@ def add_instance_to_request(instance, created, **kwargs):
     request = get_current_request()
     if request is None:
         return
+    if getattr(instance.asset.asset, 'id', None) is None:
+        # if an XForm doesn't have a real associated Asset, ignore it
+        return
     if getattr(request, 'instances', None) is None:
         request.instances = {}
+    if getattr(request, 'asset', None) is None:
+        request.asset = instance.asset.asset
     username = instance.user.username if instance.user else None
     request.instances.update(
         {

kobo/apps/audit_log/tests/test_project_history_logs.py

@@ -3,11 +3,13 @@
 import json
 import uuid
 from unittest.mock import patch
+from xml.etree import ElementTree as ET
 
 import jsonschema.exceptions
 import responses
 from ddt import data, ddt, unpack
 from django.conf import settings
+from django.contrib.auth.models import AnonymousUser
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.test import override_settings
 from django.urls import reverse
@@ -19,6 +21,7 @@
 from kobo.apps.audit_log.tests.test_models import BaseAuditLogTestCase
 from kobo.apps.hook.models import Hook
 from kobo.apps.kobo_auth.shortcuts import User
+from kobo.apps.openrosa.libs.utils.logger_tools import dict2xform
 from kpi.constants import (
     ASSET_TYPE_TEMPLATE,
     CLONE_ARG_NAME,
@@ -1660,3 +1663,60 @@ def test_multiple_submision_validation_statuses(self):
         self._check_common_metadata(log2.metadata, PROJECT_HISTORY_LOG_PROJECT_SUBTYPE)
         self.assertEqual(log2.action, AuditAction.MODIFY_SUBMISSION)
         self.assertEqual(log2.metadata['submission']['status'], 'On Hold')
+
+    @data(
+        # submit as anonymous?, use v1 endpoint?
+        (True, False),
+        (False, True),
+        (False, False),
+    )
+    @unpack
+    def test_add_submission(self, anonymous, v1):
+        # prepare submission data
+        uuid_ = uuid.uuid4()
+        self.asset.deploy(backend='mock')
+        submission_data = {
+            'q1': 'answer',
+            'q2': 'answer',
+            'meta': {'instanceID': f'uuid:{uuid_}'},
+            'formhub': {'uuid': self.asset.deployment.xform.uuid},
+            '_uuid': str(uuid_),
+        }
+        xml = ET.fromstring(
+            dict2xform(submission_data, self.asset.deployment.xform.id_string)
+        )
+        xml.tag = self.asset.uid
+        xml.attrib = {
+            'id': self.asset.uid,
+            'version': self.asset.latest_version.uid,
+        }
+        endpoint = 'submissions-list' if v1 else 'submissions'
+        kwargs = {'username': self.user.username} if not v1 else {}
+        url = reverse(
+            self._get_endpoint(endpoint),
+            kwargs=kwargs,
+        )
+        data = {'xml_submission_file': SimpleUploadedFile('name.txt', ET.tostring(xml))}
+        # ensure anonymous users are allowed to submit
+        self.asset.assign_perm(perm=PERM_ADD_SUBMISSIONS, user_obj=AnonymousUser())
+
+        if not anonymous:
+            # the submission endpoints don't allow session authentication, so
+            # just force the request to attach the correct user
+            self.client.force_authenticate(user=self.user)
+
+        # can't use _base_project_history_log_test here because our format is xml,
+        # not json
+        self.client.post(
+            url,
+            data=data,
+        )
+        logs = ProjectHistoryLog.objects.filter(metadata__asset_uid=self.asset.uid)
+        self.assertEqual(logs.count(), 1)
+        log = logs.first()
+
+        self.assertEqual(log.object_id, self.asset.id)
+        self.assertEqual(log.action, AuditAction.ADD_SUBMISSION)
+        self._check_common_metadata(log.metadata, PROJECT_HISTORY_LOG_PROJECT_SUBTYPE)
+        username = 'AnonymousUser' if anonymous else self.user.username
+        self.assertEqual(log.metadata['submission']['submitted_by'], username)

kobo/apps/audit_log/views.py

@@ -472,105 +472,109 @@ class AllProjectHistoryLogViewSet(AuditLogViewSet):
 
     **Filterable fields by action:**
 
-    1. add-media
+    * add-media
 
         a. metadata__asset-file__uid
 
         b. metadata__asset-file__filename
 
-    2. archive
+    * add-submission
+
+        a. metadata__submission__submitted_by
+
+    * archive
 
         a. metadata__latest_version_uid
 
-    3. clone-permissions
+    * clone-permissions
 
         a. metadata__cloned_from
 
-    4. connect-project
+    * connect-project
 
         a. metadata__paired-data__source_uid
 
         b. metadata__paired-data__source_name
 
-    5. delete-media
+    * delete-media
 
         a. metadata__asset-file__uid
 
         b. metadata__asset-file__filename
 
-    6. delete-service
+    * delete-service
 
         a. metadata__hook__uid
 
         b. metadata__hook__endpoint
 
         c. metadata__hook__active
 
-    7. deploy
+    * deploy
 
         a. metadata__latest_version_uid
 
         b. metadata__latest_deployed_version_uid
 
-    8. disconnect-project
+    * disconnect-project
 
         a. metadata__paired-data__source_uid
 
         b. metadata__paired-data__source_name
 
-    9. modify-imported-fields
+    * modify-imported-fields
 
         a. metadata__paired-data__source_uid
 
         b. metadata__paired-data__source_name
 
-    10. modify-service
+    * modify-service
 
         a. metadata__hook__uid
 
         b. metadata__hook__endpoint
 
         c. metadata__hook__active
 
-    11. modify-submission
+    * modify-submission
 
         a. metadata__submission__submitted_by
 
         b. metadata__submission__status (only present if changed)
 
-    12. modify-user-permissions
+    * modify-user-permissions
 
         a. metadata__permissions__username
 
-    13. redeploy
+    * redeploy
 
         a. metadata__latest_version_uid
 
         b. metadata__latest_deployed_version_uid
 
-    14. register-service
+    * register-service
 
         a. metadata__hook__uid
 
         b. metadata__hook__endpoint
 
         c. metadata__hook__active
 
-    15. transfer
+    * transfer
 
         a. metadata__username
 
-    16. unarchive
+    * unarchive
 
         a. metadata__latest_version_uid
 
-    17. update-name
+    * update-name
 
         a. metadata__name__old
 
         b. metadata__name__new
 
-    18. update-settings
+    * update-settings
 
         a. metadata__settings__description__old
 
@@ -730,105 +734,109 @@ class ProjectHistoryLogViewSet(
 
     **Filterable fields by action:**
 
-    1. add-media
+    * add-media
 
         a. metadata__asset-file__uid
 
         b. metadata__asset-file__filename
 
-    2. archive
+    * add-submission
+
+        a. metadata__submission__submitted_by
+
+    * archive
 
         a. metadata__latest_version_uid
 
-    3. clone-permissions
+    * clone-permissions
 
         a. metadata__cloned_from
 
-    4. connect-project
+    * connect-project
 
         a. metadata__paired-data__source_uid
 
         b. metadata__paired-data__source_name
 
-    5. delete-media
+    * delete-media
 
         a. metadata__asset-file__uid
 
         b. metadata__asset-file__filename
 
-    6. delete-service
+    * delete-service
 
         a. metadata__hook__uid
 
         b. metadata__hook__endpoint
 
         c. metadata__hook__active
 
-    7. deploy
+    * deploy
 
         a. metadata__latest_version_uid
 
         b. metadata__latest_deployed_version_uid
 
-    8. disconnect-project
+    * disconnect-project
 
         a. metadata__paired-data__source_uid
 
         b. metadata__paired-data__source_name
 
-    9. modify-imported-fields
+    * modify-imported-fields
 
         a. metadata__paired-data__source_uid
 
         b. metadata__paired-data__source_name
 
-    10. modify-service
+    * modify-service
 
         a. metadata__hook__uid
 
         b. metadata__hook__endpoint
 
         c. metadata__hook__active
 
-    11. modify-submission
+    * modify-submission
 
         a. metadata__submission__submitted_by
 
         b. metadata__submission__status (only present if changed)
 
-    12. modify-user-permissions
+    * modify-user-permissions
 
         a. metadata__permissions__username
 
-    13. redeploy
+    * redeploy
 
         a. metadata__latest_version_uid
 
         b. metadata__latest_deployed_version_uid
 
-    14. register-service
+    * register-service
 
         a. metadata__hook__uid
 
         b. metadata__hook__endpoint
 
         c. metadata__hook__active
 
-    15. transfer
+    * transfer
 
         a. metadata__username
 
-    16. unarchive
+    * unarchive
 
         a. metadata__latest_version_uid
 
-    17. update-name
+    * update-name
 
         a. metadata__name__old
 
         b. metadata__name__new
 
-    18. update-settings
+    * update-settings
 
         a. metadata__settings__description__old