HEX
Server: LiteSpeed
System: Linux houston.panomity.com 6.8.0-100-generic #100-Ubuntu SMP PREEMPT_DYNAMIC Tue Jan 13 16:40:06 UTC 2026 x86_64
User: nudepix (1011)
PHP: 7.4.33
Disabled: pcntl_alarm,pcntl_fork,pcntl_waitpid,pcntl_wait,pcntl_wifexited,pcntl_wifstopped,pcntl_wifsignaled,pcntl_wifcontinued,pcntl_wexitstatus,pcntl_wtermsig,pcntl_wstopsig,pcntl_signal,pcntl_signal_get_handler,pcntl_signal_dispatch,pcntl_get_last_error,pcntl_strerror,pcntl_sigprocmask,pcntl_sigwaitinfo,pcntl_sigtimedwait,pcntl_exec,pcntl_getpriority,pcntl_setpriority,pcntl_async_signals,pcntl_unshare,
Upload Files
File: //opt/moodle-mlbackend-python/moodlemlbackend/webapp/s3.py
import logging
import shutil
import tempfile
import os
import zipfile

from functools import wraps, update_wrapper

import boto3
from botocore.exceptions import ClientError

from moodlemlbackend.webapp.util import get_request_value, zipdir


# This will be set overwritten below.
localbasedir = None


class S3(object):

    def get_localbasedir(self):
        if self.localbasedir is None:
            raise Exception('localbasedir is not set')

        return self.localbasedir

    def set_localbasedir(self, basedir):
        self.localbasedir = basedir

    def get_model_dir(self, hashkey, fetch_model=False):
        '''Returns the model dir in the local fs for the provided key.

        Syncs the model with S3 if required.'''

        # No need to include uniquemodelid here, because this is all lives in
        # a temp dir in the local file system.
        modeldir = os.path.join(self.get_localbasedir(), hashkey)

        if fetch_model:

            s3 = boto3.client('s3')

            # Download the files for the provided uniquemodelid + modelhash
            bucketname = os.environ["MOODLE_MLBACKEND_PYTHON_S3_BUCKET_NAME"]
            objectkey = self.object_key(hashkey)

            # TODO Check if we should be using TemporaryFile instead.
            classifierzip = tempfile.NamedTemporaryFile()
            classifierdir = os.path.join(modeldir, 'classifier')
            try:
                s3.download_fileobj(bucketname, objectkey, classifierzip)

                if os.path.getsize(classifierzip.name) > 0:
                    with zipfile.ZipFile(classifierzip, 'r') as zipobject:

                        # The classifier directory is automatically created in
                        # moodlemlbackend.estimator but we need to create it
                        # before that point as we want to copy the classifier
                        # from S3.
                        try:
                            os.makedirs(classifierdir)
                        except FileExistsError:
                            # It can exist in some cases.
                            pass
                        zipobject.extractall(classifierdir)
            except ClientError:
                # No worries, it may perfectly not exist.
                pass

        return modeldir

    def delete_dir(self):

        s3 = boto3.resource('s3')

        bucketname = os.environ["MOODLE_MLBACKEND_PYTHON_S3_BUCKET_NAME"]
        bucket = s3.Bucket(bucketname)

        # Objectkey will equal uniquemodelid so we delete all files matching
        # uniquemodelid/ namespace.
        objectkey = self.object_key(False)

        bucket.objects.filter(Prefix=objectkey + '/').delete()

    def object_key(self, hashkey=False):

        uniquemodelid = get_request_value('uniqueid')

        if hashkey is False:
            return uniquemodelid

        dirhash = get_request_value(hashkey)
        return os.path.join(uniquemodelid, dirhash)


class S3_setup_base_dir(object):
    '''Sets the localbasedir to /tmp'''

    def __init__(self, storage, fetch_model, push_model):
        '''Sets the base dir to a temp directory.

        It fetches the requested model from s3 if required.'''

        self.storage = storage
        self.fetch_model = fetch_model
        self.push_model = push_model

        # It is our responsibility to delete this directory. However, we are
        # relying on the OS to delete it if there is any exception during the
        # course of the request.
        self.storage.set_localbasedir(tempfile.mkdtemp())

    def __call__(self, f):

        @wraps(f)
        def wrapper(*args, **kwargs):
            '''Execute the decorated function.

            Upload the model to s3 if required.'''

            update_wrapper(self, f)

            self.modeldir = self.storage.get_model_dir(
                'dirhash', fetch_model=self.fetch_model)

            # Execute the requested action.
            funcreturn = f(*args, **kwargs)

            if self.push_model is True:
                # Push the model to s3.

                s3 = boto3.client('s3')

                classifierdir = os.path.join(self.modeldir, 'classifier')

                # Copy the classifier in the model dir to S3.
                updatedclassifierzip = tempfile.NamedTemporaryFile()
                zipdir(classifierdir, updatedclassifierzip)

                # We are only interested in the model we just trained.
                bucketname = os.environ[
                    "MOODLE_MLBACKEND_PYTHON_S3_BUCKET_NAME"]
                objectkey = self.storage.object_key('dirhash')

                # Upload to S3.
                try:
                    s3.upload_file(
                        updatedclassifierzip.name, bucketname, objectkey)
                except ClientError as e:
                    # We don't want the error details in moodle as they could
                    # contain sensitive information.
                    logging.error('Error uploading the model to S3: ' + str(e))
                    return 'Can\'t upload classifier to S3.', 500

                # TODO Think about copying the new logs to S3.

            # It is our responsibility to delete tmp directories created with
            # mkdtemp
            shutil.rmtree(self.storage.get_localbasedir(), True)

            # Now that the files are copied back to S3 we can return f's
            # Response.
            return funcreturn

        return wrapper