Fix performance issue for users with many orders

justfile

@@ -4,5 +4,6 @@ lint:
 
 reset_db:
     rm -f aps_db.db
-    uv run alembic upgrade heads
+    uv run alembic upgrade 958d7aee2b21
-    for file in db_fixtures/*.sql; do sqlite3 aps_db.db < "$file"; done
+    for file in db_fixtures/*.sql; do sqlite3 aps_db.db < "$file"; done
+    uv run alembic upgrade heads

src/allmende_payment_system/api/admin.py

@@ -4,6 +4,7 @@ from typing import Annotated
 from fastapi import APIRouter, File, Form, HTTPException, Request
 from sqlalchemy import func, select
 from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import selectinload
 from starlette import status
 from starlette.responses import RedirectResponse
 
@@ -12,6 +13,7 @@ from allmende_payment_system.api.dependencies import SessionDep, UserDep
 from allmende_payment_system.models import (
     Account,
     Area,
+    Order,
     Permission,
     Product,
     ProductDetails,
@@ -310,7 +312,13 @@ async def get_accounts(
 
     templates = get_jinja_renderer()
 
-    accounts = session.scalars(select(Account)).all()
+    accounts = session.scalars(
+        select(Account).options(
+            selectinload(Account.transactions)
+            .selectinload(Transaction.order)
+            .selectinload(Order.items)
+        )
+    ).all()
     users = session.scalars(select(User)).all()
 
     return templates.TemplateResponse(

src/allmende_payment_system/api/shop.py

@@ -85,8 +85,8 @@ async def add_to_cart(request: Request, session: SessionDep, user: UserDep):
     order_item = OrderItem(
         product=product, quantity=quantity, total_amount=total_amount
     )
-    session.add(order_item)
     cart = user.shopping_cart
+    session.add(order_item)
     cart.items.append(order_item)
 
     session.flush()

src/allmende_payment_system/cli.py

@@ -81,8 +81,13 @@ def import_csv(filepath: str):
         lambda: defaultdict(lambda: Order(user_id=1))
     )
 
-    for row in rows:
+    for i, row in enumerate(rows, start=2):
         transaction_type = row["typ"].lower()
+        if transaction_type == "einkauf foodcoop":
+            print(
+                "Skipping 'Einkauf Foodcoop' transaction as it's not relevant for the payment system."
+            )
+            continue
         if transaction_type not in [
             "essen",
             "essen kind",
@@ -90,8 +95,9 @@ def import_csv(filepath: str):
             "einkauf",
             "auszahlung",
         ]:
-            # TODO: Handle other types
+            raise ValueError(
-            continue
+                f"Skipping row {i} with unknown transaction type: '{transaction_type}'"
+            )
 
         # find or create account for "Partei/Konto"
         account_name = row["partei_konto"]
@@ -123,7 +129,7 @@ def import_csv(filepath: str):
                         if transaction_type == "essen"
                         else kids_meal_product
                     ),
-                    total_amount=row["betrag"],
+                    total_amount=row["betrag"] * -1,
                     quantity=quantity,
                 )
             )

src/allmende_payment_system/models.py

@@ -2,7 +2,7 @@ import datetime
 import decimal
 import typing
 
-from sqlalchemy import Column, ForeignKey, Numeric, Table, and_, func, select
+from sqlalchemy import Column, ForeignKey, Numeric, Table, and_, exists, func, select
 from sqlalchemy.exc import NoResultFound
 from sqlalchemy.orm import (
     DeclarativeBase,
@@ -35,8 +35,6 @@ class Account(Base):
 
     @property
     def balance(self):
-        for t in self.transactions:
-            print(t)
         return sum(t.total_amount for t in self.transactions)
 
 
@@ -49,6 +47,7 @@ user_account_association = Table(
 
 
 class User(Base):
+
     __tablename__ = TABLE_PREFIX + "user"
     id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
     username: Mapped[str] = mapped_column(nullable=False, unique=True)
@@ -57,7 +56,11 @@ class User(Base):
     accounts: Mapped[list["Account"]] = relationship(
         "Account", secondary=user_account_association, back_populates="users"
     )
-    orders: Mapped[list["Order"]] = relationship("Order", back_populates="user")
+    orders: Mapped[list["Order"]] = relationship(
+        "Order",
+        back_populates="user",
+        lazy="selectin",
+    )
 
     user_groups: Mapped[list["UserGroup"]] = relationship(
         "UserGroup",
@@ -65,18 +68,29 @@ class User(Base):
         back_populates="users",
     )
 
+    _shopping_cart: Mapped["Order"] = relationship(
+        "Order",
+        uselist=False,
+        primaryjoin=lambda: and_(
+            User.id == Order.user_id,
+            ~exists().where(Transaction.order_id == Order.id),
+        ),
+        viewonly=True,
+    )
+
     @property
     def shopping_cart(self):
-        for order in self.orders:
+        if self._shopping_cart is None:
-            if order.transaction is None:
-                cart = order
-                break
-        else:
-            cart = Order(user=self)
             session = object_session(self)
-            session.add(cart)
+            assert session is not None, "User object is not attached to a session."
 
-        return cart
+            cart = Order(user_id=self.id)
+            self.orders.append(cart)
+            session.add(cart)
+            session.flush()
+            session.refresh(self, ["_shopping_cart"])
+
+        return self._shopping_cart
 
     def has_permission(self, scope: str, action: str) -> bool:
         for group in self.user_groups:
@@ -236,13 +250,15 @@ class Order(Base):
         # create a transaction for the order
         transaction = Transaction(
             type="order",
-            total_amount=-self.total_amount,
             order=self,
             account=account,
         )
         session = object_session(self)
         session.add(transaction)
 
+        # expire the shopping cart relationship to force it to be reloaded and show that the order is no longer in the shopping cart
+        session.expire(self.user, ["_shopping_cart"])
+
 
 class OrderItem(Base):
     __tablename__ = TABLE_PREFIX + "order_item"
@@ -296,5 +312,13 @@ class Transaction(Base):
     @property
     def total_amount(self):
         if self.type == "order" and self.order is not None:
-            return self.order.total_amount
+            return self.order.total_amount * -1
         return self._total_amount
+
+    @total_amount.setter
+    def total_amount(self, value):
+        if self.type == "order":
+            raise AttributeError(
+                "Cannot set total_amount of an order transaction directly."
+            )
+        self._total_amount = value

src/allmende_payment_system/tools.py

@@ -30,3 +30,78 @@ def get_jinja_renderer() -> Jinja2Templates:
 
     renderer.env.globals["production_mode"] = "APS_PRODUCTION_MODE" in os.environ
     return renderer
+
+
+def timer_context(name: str):
+    import time
+
+    class Timer:
+        def __enter__(self):
+            self.start = time.perf_counter()
+
+        def __exit__(self, exc_type, exc_val, exc_tb):
+            end = time.perf_counter()
+            print(f"{name} took {end - self.start:.4f} seconds")
+
+    return Timer()
+
+
+import cProfile, pstats, io
+from contextlib import contextmanager
+
+
+@contextmanager
+def profile(top_n: int = 20, sort_by: str = "cumulative", min_time: float = 0.001):
+    pr = cProfile.Profile()
+    pr.enable()
+    try:
+        yield
+    finally:
+        pr.disable()
+        s = io.StringIO()
+        ps = pstats.Stats(pr, stream=s).strip_dirs()
+        ps.sort_stats(sort_by)
+        ps.print_stats()
+
+        rows = []
+        for line in s.getvalue().splitlines():
+            parts = line.split(None, 5)
+            if len(parts) < 6:
+                continue
+            try:
+                cum = float(parts[3])
+                if cum >= min_time:
+                    rows.append(
+                        (
+                            int(parts[0].split("/")[0]),
+                            float(parts[1]),
+                            cum,
+                            float(parts[4]),
+                            parts[5].strip(),
+                        )
+                    )
+            except (ValueError, IndexError):
+                continue
+
+        rows = rows[:top_n]
+        if not rows:
+            print("[profiler] nothing exceeded min_time threshold")
+            return
+
+        w = 55
+        print(f"\n── profile ({sort_by}, ≥{min_time}s) ──────────────────────────────")
+        print(f"{'calls':>7} {'tot':>8} {'cum':>8} {'percall':>8}  {'function':<{w}}")
+        print("─" * (7 + 8 + 8 + 8 + w + 6))
+        for calls, tot, cum, percall, loc in rows:
+            c = (
+                f"\033[91m{cum:8.4f}\033[0m"
+                if cum > 1
+                else f"\033[93m{cum:8.4f}\033[0m" if cum > 0.1 else f"{cum:8.4f}"
+            )
+            print(
+                f"{calls:>7} {tot:8.4f} {c} {percall:8.4f}  {loc[-w:] if len(loc) > w else loc:<{w}}"
+            )
+        print(f"─" * (7 + 8 + 8 + 8 + w + 6))
+        print(
+            f"  {len(rows)} functions | tot accounted: {sum(r[1] for r in rows):.4f}s\n"
+        )