Skip to content

rs_workflows/operation/quota_monitoring_db_models.md

<< Back to index

Tables for the quota monitoring database

Base

Bases: DeclarativeBase

Base class for SQLAlchemy declarative models.

Source code in docs/rs-client-libraries/rs_workflows/operation/quota_monitoring_db_models.py 
22
23
class Base(DeclarativeBase):  # pylint: disable=too-few-public-methods
    """Base class for SQLAlchemy declarative models."""

S3AccessLog

Bases: Base

ORM mapping for the s3_access_log table. Mirrors the PostgreSQL schema.

Source code in docs/rs-client-libraries/rs_workflows/operation/quota_monitoring_db_models.py 
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
class S3AccessLog(Base):  # pylint: disable=too-few-public-methods
    """
    ORM mapping for the s3_access_log table.
    Mirrors the PostgreSQL schema.
    """

    __tablename__ = "s3_access_log"

    id = Column(BigInteger, primary_key=True, autoincrement=True)
    bucket_owner = Column(Text)
    bucket = Column(Text, nullable=False)
    time = Column(TIMESTAMP, nullable=False)
    remote_ip = Column(INET)
    requester = Column(Text)
    request_id = Column(Text)
    operation = Column(Text)
    key = Column(Text)
    request_uri = Column(Text)
    http_status = Column(Integer)
    error_code = Column(Text)
    bytes_sent = Column(BigInteger)
    object_size = Column(BigInteger)
    total_time_ms = Column(Integer)
    turnaround_time_ms = Column(Integer)
    referer = Column(Text)
    user_agent = Column(Text)
    version_id = Column(Text)
    signature_version = Column(Text)
    authentication_type = Column(Text)
    host_header = Column(Text)

    __table_args__ = (
        # Composite index to speed up queries filtering on recent logs
        # and grouping by bucket (e.g., WHERE time >= NOW() - INTERVAL '30 days')
        Index("idx_s3log_time_bucket", "time", "bucket"),
        # Composite index to accelerate queries filtering by time and requester
        Index("idx_s3log_recent_requester", "time", "requester"),
        # Partial index for PUT operations (REST.PUT.PART)
        # Optimizes queries involving object_size for multipart uploads
        Index(
            "idx_s3log_put",
            "object_size",
            postgresql_where=(operation == "REST.PUT.PART"),
        ),
        # Partial index for GET operations (REST.GET.OBJECT)
        # Optimizes queries involving bytes_sent for object downloads
        Index(
            "idx_s3log_get",
            "bytes_sent",
            postgresql_where=(operation == "REST.GET.OBJECT"),
        ),
    )

S3LogConsolidate

Bases: Base

ORM mapping for the s3_log_consolidate table. Mirrors the PostgreSQL schema.

Source code in docs/rs-client-libraries/rs_workflows/operation/quota_monitoring_db_models.py 
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
class S3LogConsolidate(Base):  # pylint: disable=too-few-public-methods
    """
    ORM mapping for the s3_log_consolidate table.
    Mirrors the PostgreSQL schema.
    """

    __tablename__ = "s3_log_consolidate"

    id = Column(BigInteger, primary_key=True, autoincrement=True)

    bucket = Column(Text, nullable=False)
    time = Column(TIMESTAMP, nullable=False)
    requester = Column(Text)
    operation = Column(Text)

    bytes_sent = Column(BigInteger)
    object_size = Column(BigInteger)
    total_time_ms = Column(BigInteger)
    turnaround_time_ms = Column(BigInteger)

    created_at = Column(TIMESTAMP, nullable=False, server_default=text("now()"))

    __table_args__ = (
        # Index to accelerate recent-time queries by bucket
        Index("idx_s3log_consolidate_time_bucket", "time", "bucket"),
        # Composite index for analytics by requester and operation
        Index("idx_s3log_consolidate_time_requester_operation", "time", "requester", "operation"),
    )