Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions contentcuration/contentcuration/constants/locking.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,5 @@
"""
TREE_LOCK = 1001
TASK_LOCK = 1002
CHANNEL_CHANGE_LOCK = 1003
USER_CHANGE_LOCK = 1004
21 changes: 17 additions & 4 deletions contentcuration/contentcuration/db/advisory_lock.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,18 +88,31 @@ def advisory_lock(key1, key2=None, shared=False):
pass


def try_advisory_lock(key1, key2=None, shared=False):
def try_advisory_lock(key1, key2=None, shared=False, session=False):
"""
Creates a transaction level advisory lock that doesn't block

:param key1: int
:param key2: int
:param shared: bool
:raises: AdvisoryLockBusy
"""
with execute_lock(key1, key2=key2, shared=shared, wait=False) as cursor:
with execute_lock(
key1, key2=key2, shared=shared, session=session, wait=False
) as cursor:
# for `try` locks, the PG function will return True or False,
# representing whether the lock was acquired or not
results = cursor.fetchone()
if not results[0]:
raise AdvisoryLockBusy("Unable to acquire advisory lock")


@contextmanager
def try_session_advisory_lock(key1, key2=None):
"""
:raises: AdvisoryLockBusy
"""
try_advisory_lock(key1, key2=key2, session=True)
try:
yield
finally:
with execute_lock(key1, key2=key2, unlock=True, session=True):
pass
41 changes: 31 additions & 10 deletions contentcuration/contentcuration/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
"""
import logging
import time
import zlib
from contextlib import ExitStack

from celery.utils.log import get_task_logger
from django.conf import settings
Expand All @@ -13,6 +15,10 @@
from django.utils.translation import override

from contentcuration.celery import app
from contentcuration.constants.locking import CHANNEL_CHANGE_LOCK
from contentcuration.constants.locking import USER_CHANGE_LOCK
from contentcuration.db.advisory_lock import AdvisoryLockBusy
from contentcuration.db.advisory_lock import try_session_advisory_lock
from contentcuration.models import Change
from contentcuration.models import ContentNode
from contentcuration.models import User
Expand All @@ -25,20 +31,37 @@
logger = get_task_logger(__name__)


def channel_lock_key(channel_id):
# crc32 collisions let two channels share a lock, at odds of 2**-32 per pair
# of channels with tasks running at once; the loser's changes wait for that
# channel's next sync.
return zlib.crc32(channel_id.encode("utf-8"))


def _apply_changes_exclusively(task, changes_qs, key1, key2):
from contentcuration.viewsets.sync.base import apply_changes

with ExitStack() as stack:
try:
stack.enter_context(try_session_advisory_lock(key1, key2=key2))
except AdvisoryLockBusy:
logger.info(f"Skipping {task.name}: another task holds lock {key1}, {key2}")
return
apply_changes(changes_qs)
if changes_qs.exists():
task.requeue()


@app.task(bind=True, name="apply_user_changes")
def apply_user_changes_task(self, user_id):
"""
:type self: contentcuration.utils.celery.tasks.CeleryTask
:param user_id: The user ID for which to process changes
"""
from contentcuration.viewsets.sync.base import apply_changes

changes_qs = Change.objects.filter(
applied=False, errored=False, user_id=user_id, channel__isnull=True
)
apply_changes(changes_qs)
if changes_qs.exists():
self.requeue()
_apply_changes_exclusively(self, changes_qs, USER_CHANGE_LOCK, user_id)


@app.task(bind=True, name="apply_channel_changes")
Expand All @@ -47,14 +70,12 @@ def apply_channel_changes_task(self, channel_id):
:type self: contentcuration.utils.celery.tasks.CeleryTask
:param channel_id: The channel ID for which to process changes
"""
from contentcuration.viewsets.sync.base import apply_changes

changes_qs = Change.objects.filter(
applied=False, errored=False, channel_id=channel_id
)
apply_changes(changes_qs)
if changes_qs.exists():
self.requeue()
_apply_changes_exclusively(
self, changes_qs, CHANNEL_CHANGE_LOCK, channel_lock_key(channel_id)
)


class CustomEmailMessage(EmailMessage):
Expand Down
33 changes: 33 additions & 0 deletions contentcuration/contentcuration/tests/db/test_advisory_lock.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from multiprocessing import set_start_method
from time import sleep

from django.db import connection
from django.db import transaction
from django.test.testcases import SimpleTestCase
from django_concurrent_tests.management.commands.concurrent_call_wrapper import (
Expand All @@ -20,6 +21,7 @@
from contentcuration.db.advisory_lock import AdvisoryLockBusy
from contentcuration.db.advisory_lock import execute_lock
from contentcuration.db.advisory_lock import try_advisory_lock
from contentcuration.db.advisory_lock import try_session_advisory_lock

TEST_LOCK = 1337

Expand Down Expand Up @@ -330,3 +332,34 @@ def test_try__busy(self):
# since the lock should already be acquired, this will raise the error
with raises(AdvisoryLockBusy):
try_advisory_lock(TEST_LOCK)

def assert_advisory_locks_held(self, *locks):
with connection.cursor() as c:
c.execute(
"SELECT classid::bigint, objid::bigint, objsubid FROM pg_locks"
" WHERE locktype = 'advisory' AND pid = pg_backend_pid()"
)
self.assertEqual(sorted(c.fetchall()), sorted(locks))

@mark.timeout(30)
def test_try_session__busy(self):
with self.child_lock(shared=False):
with raises(AdvisoryLockBusy):
with try_session_advisory_lock(TEST_LOCK):
pass

@mark.timeout(30)
def test_try_session__held_across_commits(self):
with try_session_advisory_lock(TEST_LOCK, key2=1):
self.assertFalse(connection.in_atomic_block)
with transaction.atomic():
pass
self.assert_advisory_locks_held((TEST_LOCK, 1, 2))
self.assert_advisory_locks_held()

@mark.timeout(30)
def test_try_session__released_on_exception(self):
with raises(ValueError):
with try_session_advisory_lock(TEST_LOCK, key2=1):
raise ValueError()
self.assert_advisory_locks_held()
9 changes: 6 additions & 3 deletions contentcuration/contentcuration/tests/test_asynctask.py
Original file line number Diff line number Diff line change
Expand Up @@ -241,15 +241,18 @@ def test_fetch_or_enqueue_task__channel_id__uuid_then_hex(self):
self.assertEqual(expected_task.task_id, async_result.task_id)

def test_requeue_task(self):
signature = requeue_test_task.generate_signature({})
existing_task_ids = requeue_test_task.find_ids(signature)
self.assertEqual(len(existing_task_ids), 0)
requeued_signature = requeue_test_task.generate_signature(
{"requeue": True, "ran": True}
)
self.assertEqual(len(requeue_test_task.find_ids(requeued_signature)), 0)

first_async_result = requeue_test_task.enqueue(self.user, requeue=True)
first_result = self._wait_for(first_async_result)
self.assertIsNotNone(first_result)
self.assertTrue(first_async_result.successful())

self.assertIn(first_result, requeue_test_task.find_ids(requeued_signature))

second_async_result = requeue_test_task.fetch(first_result)
# assert that the requeued task was requeued by the same user
self.assertEqual(second_async_result.user_id, self.user.id)
Expand Down
156 changes: 156 additions & 0 deletions contentcuration/contentcuration/tests/test_tasks.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
import uuid
from contextlib import contextmanager

from django.db import connection
from django.db import connections
from le_utils.constants import content_kinds

from contentcuration import models
from contentcuration.constants.locking import CHANNEL_CHANGE_LOCK
from contentcuration.constants.locking import USER_CHANGE_LOCK
from contentcuration.db.advisory_lock import _prepare_keys
from contentcuration.tasks import apply_channel_changes_task
from contentcuration.tasks import apply_user_changes_task
from contentcuration.tasks import channel_lock_key
from contentcuration.tests import testdata
from contentcuration.tests.base import StudioTestCase
from contentcuration.tests.viewsets.base import generate_create_event
from contentcuration.tests.viewsets.base import SyncTestMixin
from contentcuration.viewsets.sync.constants import BOOKMARK
from contentcuration.viewsets.sync.constants import CONTENTNODE


@contextmanager
def lock_held_elsewhere(key1, key2):
other = connections.create_connection("default")
try:
with other.cursor() as c:
c.execute("SELECT pg_advisory_lock(%s, %s)", _prepare_keys([key1, key2]))
yield
finally:
other.close()


def lock_is_free(key1, key2):
other = connections.create_connection("default")
try:
with other.cursor() as c:
c.execute(
"SELECT pg_try_advisory_lock(%s, %s)", _prepare_keys([key1, key2])
)
return c.fetchone()[0]
finally:
other.close()


class ApplyChangesTaskTestCase(SyncTestMixin, StudioTestCase):
def setUp(self):
super().setUp()
self.user = testdata.user()
self.channel = testdata.channel()
self.channel.editors.add(self.user)
self.other_channel = testdata.channel()
self.other_channel.editors.add(self.user)

def create_node_change(self, channel):
node_id = uuid.uuid4().hex
change = models.Change.create_change(
generate_create_event(
node_id,
CONTENTNODE,
{
"title": "node",
"id": node_id,
"kind": content_kinds.TOPIC,
"parent": channel.main_tree_id,
},
channel_id=channel.id,
),
created_by_id=self.user.id,
)
return change, node_id

def create_bookmark_change(self):
return models.Change.create_change(
generate_create_event(
self.channel.id,
BOOKMARK,
{"channel": self.channel.id},
user_id=self.user.id,
),
created_by_id=self.user.id,
)

def run_channel_task(self, channel):
apply_channel_changes_task.fetch_or_enqueue(self.user, channel_id=channel.id)

def run_user_task(self):
apply_user_changes_task.fetch_or_enqueue(self.user, user_id=self.user.id)

def test_channel_lock__excludes_only_its_own_channel(self):
locked_change, locked_node_id = self.create_node_change(self.channel)
other_change, other_node_id = self.create_node_change(self.other_channel)
self.create_bookmark_change()

with lock_held_elsewhere(
CHANNEL_CHANGE_LOCK, channel_lock_key(self.channel.id)
):
self.run_channel_task(self.channel)
self.run_channel_task(self.other_channel)
self.run_user_task()

self.assertEqual(models.CustomTaskMetadata.objects.count(), 3)
locked_change.refresh_from_db()
self.assertFalse(locked_change.applied)
self.assertFalse(locked_change.errored)
self.assertFalse(models.ContentNode.objects.filter(id=locked_node_id).exists())
other_change.refresh_from_db()
self.assertTrue(other_change.applied)
self.assertTrue(models.ContentNode.objects.filter(id=other_node_id).exists())
self.assertTrue(
self.user.bookmarked_channels.filter(id=self.channel.id).exists()
)

def test_user_lock__excludes_its_own_user(self):
change = self.create_bookmark_change()

with lock_held_elsewhere(USER_CHANGE_LOCK, self.user.id):
self.run_user_task()

self.assertEqual(models.CustomTaskMetadata.objects.count(), 1)
change.refresh_from_db()
self.assertFalse(change.applied)
self.assertFalse(change.errored)
self.assertFalse(self.user.bookmarked_channels.exists())

def test_channel_task__holds_lock_while_applying_changes(self):
self.create_node_change(self.channel)
lock_free_during_insert = []

def probe_lock(execute, sql, params, many, context):
if not lock_free_during_insert and sql.startswith(
'INSERT INTO "contentcuration_contentnode"'
):
lock_free_during_insert.append(
lock_is_free(CHANNEL_CHANGE_LOCK, channel_lock_key(self.channel.id))
)
return execute(sql, params, many, context)

with connection.execute_wrapper(probe_lock):
self.run_channel_task(self.channel)

self.assertEqual(lock_free_during_insert, [False])

def test_channel_task__requeues_for_change_created_before_unlock(self):
self.create_node_change(self.channel)
late_node_ids = []

def create_change_before_unlock(execute, sql, params, many, context):
if not late_node_ids and "pg_advisory_unlock" in sql:
late_node_ids.append(self.create_node_change(self.channel)[1])
return execute(sql, params, many, context)

with connection.execute_wrapper(create_change_before_unlock):
self.run_channel_task(self.channel)

self.assertTrue(models.ContentNode.objects.filter(id=late_node_ids[0]).exists())
2 changes: 1 addition & 1 deletion contentcuration/contentcuration/utils/celery/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -300,7 +300,7 @@ def requeue(self, **kwargs):
)
task_kwargs = request.kwargs.copy()
task_kwargs.update(kwargs)
signature = self.generate_signature(kwargs)
signature = self.generate_signature(task_kwargs)
custom_task_metadata = CustomTaskMetadata.objects.get(task_id=request.id)
logging.info(
f"Re-queuing task {self.name} for user {custom_task_metadata.user.pk} from {request.id} | {signature}"
Expand Down
Loading