feat(dbt): Add containerised Jaffle Shop dbt project

feat(dbt): Add dbt project file

.gitignore

@@ -0,0 +1,3 @@
+target/
+dbt_modules/
+logs/

Dockerfile

@@ -0,0 +1,15 @@
+FROM --platform=linux/amd64 python:3.10-slim-buster
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends
+
+WORKDIR /usr/src/dbt
+
+# Install the dbt Postgres adapter. This step will also install dbt-core
+RUN pip install --upgrade pip
+RUN pip install dbt-postgres==1.2.0
+RUN pip install pytz
+
+# Install dbt dependencies (as specified in packages.yml file)
+# Build seeds, models and snapshots (and run tests wherever applicable)
+CMD dbt deps && dbt build --profiles-dir ./profiles && sleep infinity

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright [2023] [Giorgos Myrianthous]
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

README.md

@@ -0,0 +1,106 @@
+# Containerised Jaffle Shop dbt Project
+This is a containerised version of the popular 
+[Jaffle Shop dbt project published by dbt Labs](https://github.com/dbt-labs/jaffle_shop). 
+You can use this project to populate and run Jaffle Shop in a Postgres instance.
+
+# Running the project with Docker
+You'll first need to build and run the services via Docker (as defined in `docker-compose.yml`):
+```bash
+$ docker-compose build
+$ docker-compose up
+```
+
+The commands above will run a Postgres instance and then build the dbt resources of Jaffle Shop as specified in the
+repository. These containers will remain up and running so that you can:
+- Query the Postgres database and the tables created out of dbt models
+- Run further dbt commands via dbt CLI
+
+
+## Building additional or modified data models
+Once the containers are up and running, you can still make any modifications in the existing dbt project 
+and re-run any command to serve the purpose of the modifications. 
+
+In order to build your data models, you first need to access the container.
+
+To do so, we infer the container id for `dbt` running container:
+```bash
+docker ps
+```
+
+Then enter the running container:
+```bash
+docker exec -it <container-id> /bin/bash
+```
+
+And finally:
+
+```bash
+# Install dbt deps (might not required as long as you have no -or empty- `dbt_packages.yml` file)
+dbt deps
+
+# Build seeds
+dbt seeds --profiles-dir profiles
+
+# Build data models
+dbt run --profiles-dir profiles
+
+# Build snapshots
+dbt snapshot --profiles-dir profiles
+
+# Run tests
+dbt test --profiles-dir profiles
+```
+
+Alternatively, you can run everything in just a single command:
+
+```bash
+dbt build --profiles-dir profiles
+```
+
+## Querying Jaffle Shop data models on Postgres
+In order to query and verify the seeds, models and snapshots created in the dummy dbt project, simply follow the 
+steps below. 
+
+Find the container id of the postgres service:
+```commandline
+docker ps 
+```
+
+Then run 
+```commandline
+docker exec -it <container-id> /bin/bash
+```
+
+We will then use `psql`, a terminal-based interface for PostgreSQL that allows us to query the database:
+```commandline
+psql -U postgres
+```
+
+You can list tables and views as shown below:
+```bash
+postgres=# \dt
+             List of relations
+ Schema |     Name      | Type  |  Owner   
+--------+---------------+-------+----------
+ public | customers     | table | postgres
+ public | orders        | table | postgres
+ public | raw_customers | table | postgres
+ public | raw_orders    | table | postgres
+ public | raw_payments  | table | postgres
+(5 rows)
+
+postgres=# \dv
+            List of relations
+ Schema |     Name      | Type |  Owner   
+--------+---------------+------+----------
+ public | stg_customers | view | postgres
+ public | stg_orders    | view | postgres
+ public | stg_payments  | view | postgres
+(3 rows)
+
+```
+
+Now you can query the tables constructed form the seeds, models and snapshots defined in the dbt project:
+```sql
+SELEC * FROM <table_or_view_name>;
+```

dbt_project.yml

@@ -0,0 +1,26 @@
+name: 'jaffle_shop'
+
+config-version: 2
+version: '0.1'
+
+profile: 'jaffle_shop'
+
+model-paths: ["models"]
+seed-paths: ["seeds"]
+test-paths: ["tests"]
+analysis-paths: ["analysis"]
+macro-paths: ["macros"]
+
+target-path: "target"
+clean-targets:
+    - "target"
+    - "dbt_modules"
+    - "logs"
+
+require-dbt-version: [">=1.0.0", "<2.0.0"]
+
+models:
+  jaffle_shop:
+      materialized: table
+      staging:
+        materialized: view

docker-compose.yml

@@ -0,0 +1,25 @@
+version: "3.9"
+
+services:
+  postgres:
+    container_name: postgres
+    image: postgres:15.2-alpine
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+    ports:
+      - 5432
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+  dbt:
+    container_name: dbt
+    build: .
+    image: dbt-jaffle-shop
+    volumes:
+      - ./:/usr/src/dbt
+    depends_on:
+      postgres:
+        condition: service_healthy

models/customers.sql

@@ -0,0 +1,69 @@
+with customers as (
+
+    select * from {{ ref('stg_customers') }}
+
+),
+
+orders as (
+
+    select * from {{ ref('stg_orders') }}
+
+),
+
+payments as (
+
+    select * from {{ ref('stg_payments') }}
+
+),
+
+customer_orders as (
+
+        select
+        customer_id,
+
+        min(order_date) as first_order,
+        max(order_date) as most_recent_order,
+        count(order_id) as number_of_orders
+    from orders
+
+    group by customer_id
+
+),
+
+customer_payments as (
+
+    select
+        orders.customer_id,
+        sum(amount) as total_amount
+
+    from payments
+
+    left join orders on
+         payments.order_id = orders.order_id
+
+    group by orders.customer_id
+
+),
+
+final as (
+
+    select
+        customers.customer_id,
+        customers.first_name,
+        customers.last_name,
+        customer_orders.first_order,
+        customer_orders.most_recent_order,
+        customer_orders.number_of_orders,
+        customer_payments.total_amount as customer_lifetime_value
+
+    from customers
+
+    left join customer_orders
+        on customers.customer_id = customer_orders.customer_id
+
+    left join customer_payments
+        on  customers.customer_id = customer_payments.customer_id
+
+)
+
+select * from final

models/docs.md

@@ -0,0 +1,14 @@
+{% docs orders_status %}
+
+Orders can be one of the following statuses:
+
+| status         | description                                                                                                            |
+|----------------|------------------------------------------------------------------------------------------------------------------------|
+| placed         | The order has been placed but has not yet left the warehouse                                                           |
+| shipped        | The order has ben shipped to the customer and is currently in transit                                                  |
+| completed      | The order has been received by the customer                                                                            |
+| return_pending | The customer has indicated that they would like to return the order, but it has not yet been received at the warehouse |
+| returned       | The order has been returned by the customer and received at the warehouse                                              |
+
+
+{% enddocs %}

models/orders.sql

@@ -0,0 +1,56 @@
+{% set payment_methods = ['credit_card', 'coupon', 'bank_transfer', 'gift_card'] %}
+
+with orders as (
+
+    select * from {{ ref('stg_orders') }}
+
+),
+
+payments as (
+
+    select * from {{ ref('stg_payments') }}
+
+),
+
+order_payments as (
+
+    select
+        order_id,
+
+        {% for payment_method in payment_methods -%}
+        sum(case when payment_method = '{{ payment_method }}' then amount else 0 end) as {{ payment_method }}_amount,
+        {% endfor -%}
+
+        sum(amount) as total_amount
+
+    from payments
+
+    group by order_id
+
+),
+
+final as (
+
+    select
+        orders.order_id,
+        orders.customer_id,
+        orders.order_date,
+        orders.status,
+
+        {% for payment_method in payment_methods -%}
+
+        order_payments.{{ payment_method }}_amount,
+
+        {% endfor -%}
+
+        order_payments.total_amount as amount
+
+    from orders
+
+
+    left join order_payments
+        on orders.order_id = order_payments.order_id
+
+)
+
+select * from final

models/overview.md

@@ -0,0 +1,11 @@
+{% docs __overview__ %}
+
+## Data Documentation for Jaffle Shop
+
+`jaffle_shop` is a fictional ecommerce store.
+
+This [dbt](https://www.getdbt.com/) project is for testing out code.
+
+The source code can be found [here](https://github.com/clrcrl/jaffle_shop).
+
+{% enddocs %}

models/schema.yml

@@ -0,0 +1,82 @@
+version: 2
+
+models:
+  - name: customers
+    description: This table has basic information about a customer, as well as some derived facts based on a customer's orders
+
+    columns:
+      - name: customer_id
+        description: This is a unique identifier for a customer
+        tests:
+          - unique
+          - not_null
+
+      - name: first_name
+        description: Customer's first name. PII.
+
+      - name: last_name
+        description: Customer's last name. PII.
+
+      - name: first_order
+        description: Date (UTC) of a customer's first order
+
+      - name: most_recent_order
+        description: Date (UTC) of a customer's most recent order
+
+      - name: number_of_orders
+        description: Count of the number of orders a customer has placed
+
+      - name: total_order_amount
+        description: Total value (AUD) of a customer's orders
+
+  - name: orders
+    description: This table has basic information about orders, as well as some derived facts based on payments
+
+    columns:
+      - name: order_id
+        tests:
+          - unique
+          - not_null
+        description: This is a unique identifier for an order
+
+      - name: customer_id
+        description: Foreign key to the customers table
+        tests:
+          - not_null
+          - relationships:
+              to: ref('customers')
+              field: customer_id
+
+      - name: order_date
+        description: Date (UTC) that the order was placed
+
+      - name: status
+        description: '{{ doc("orders_status") }}'
+        tests:
+          - accepted_values:
+              values: ['placed', 'shipped', 'completed', 'return_pending', 'returned']
+
+      - name: amount
+        description: Total amount (AUD) of the order
+        tests:
+          - not_null
+
+      - name: credit_card_amount
+        description: Amount of the order (AUD) paid for by credit card
+        tests:
+          - not_null
+
+      - name: coupon_amount
+        description: Amount of the order (AUD) paid for by coupon
+        tests:
+          - not_null
+
+      - name: bank_transfer_amount
+        description: Amount of the order (AUD) paid for by bank transfer
+        tests:
+          - not_null
+
+      - name: gift_card_amount
+        description: Amount of the order (AUD) paid for by gift card
+        tests:
+          - not_null

models/staging/schema.yml

@@ -0,0 +1,31 @@
+version: 2
+
+models:
+  - name: stg_customers
+    columns:
+      - name: customer_id
+        tests:
+          - unique
+          - not_null
+
+  - name: stg_orders
+    columns:
+      - name: order_id
+        tests:
+          - unique
+          - not_null
+      - name: status
+        tests:
+          - accepted_values:
+              values: ['placed', 'shipped', 'completed', 'return_pending', 'returned']
+
+  - name: stg_payments
+    columns:
+      - name: payment_id
+        tests:
+          - unique
+          - not_null
+      - name: payment_method
+        tests:
+          - accepted_values:
+              values: ['credit_card', 'coupon', 'bank_transfer', 'gift_card']

models/staging/stg_customers.sql

@@ -0,0 +1,22 @@
+with source as (
+
+    {#-
+    Normally we would select from the table here, but we are using seeds to load
+    our data in this project
+    #}
+    select * from {{ ref('raw_customers') }}
+
+),
+
+renamed as (
+
+    select
+        id as customer_id,
+        first_name,
+        last_name
+
+    from source
+
+)
+
+select * from renamed

models/staging/stg_orders.sql

@@ -0,0 +1,23 @@
+with source as (
+
+    {#-
+    Normally we would select from the table here, but we are using seeds to load
+    our data in this project
+    #}
+    select * from {{ ref('raw_orders') }}
+
+),
+
+renamed as (
+
+    select
+        id as order_id,
+        user_id as customer_id,
+        order_date,
+        status
+
+    from source
+
+)
+
+select * from renamed

models/staging/stg_payments.sql

@@ -0,0 +1,25 @@
+with source as (
+    
+    {#-
+    Normally we would select from the table here, but we are using seeds to load
+    our data in this project
+    #}
+    select * from {{ ref('raw_payments') }}
+
+),
+
+renamed as (
+
+    select
+        id as payment_id,
+        order_id,
+        payment_method,
+
+        -- `amount` is currently stored in cents, so we convert it to dollars
+        amount / 100 as amount
+
+    from source
+
+)
+
+select * from renamed

profiles/profiles.yml

@@ -0,0 +1,12 @@
+jaffle_shop:
+  target: dev
+  outputs:
+    dev:
+      type: postgres
+      host: postgres
+      user: postgres
+      password: postgres
+      port: 5432
+      dbname: postgres
+      schema: public
+      threads: 1

seeds/raw_customers.csv

@@ -0,0 +1,101 @@
+id,first_name,last_name
+1,Michael,P.
+2,Shawn,M.
+3,Kathleen,P.
+4,Jimmy,C.
+5,Katherine,R.
+6,Sarah,R.
+7,Martin,M.
+8,Frank,R.
+9,Jennifer,F.
+10,Henry,W.
+11,Fred,S.
+12,Amy,D.
+13,Kathleen,M.
+14,Steve,F.
+15,Teresa,H.
+16,Amanda,H.
+17,Kimberly,R.
+18,Johnny,K.
+19,Virginia,F.
+20,Anna,A.
+21,Willie,H.
+22,Sean,H.
+23,Mildred,A.
+24,David,G.
+25,Victor,H.
+26,Aaron,R.
+27,Benjamin,B.
+28,Lisa,W.
+29,Benjamin,K.
+30,Christina,W.
+31,Jane,G.
+32,Thomas,O.
+33,Katherine,M.
+34,Jennifer,S.
+35,Sara,T.
+36,Harold,O.
+37,Shirley,J.
+38,Dennis,J.
+39,Louise,W.
+40,Maria,A.
+41,Gloria,C.
+42,Diana,S.
+43,Kelly,N.
+44,Jane,R.
+45,Scott,B.
+46,Norma,C.
+47,Marie,P.
+48,Lillian,C.
+49,Judy,N.
+50,Billy,L.
+51,Howard,R.
+52,Laura,F.
+53,Anne,B.
+54,Rose,M.
+55,Nicholas,R.
+56,Joshua,K.
+57,Paul,W.
+58,Kathryn,K.
+59,Adam,A.
+60,Norma,W.
+61,Timothy,R.
+62,Elizabeth,P.
+63,Edward,G.
+64,David,C.
+65,Brenda,W.
+66,Adam,W.
+67,Michael,H.
+68,Jesse,E.
+69,Janet,P.
+70,Helen,F.
+71,Gerald,C.
+72,Kathryn,O.
+73,Alan,B.
+74,Harry,A.
+75,Andrea,H.
+76,Barbara,W.
+77,Anne,W.
+78,Harry,H.
+79,Jack,R.
+80,Phillip,H.
+81,Shirley,H.
+82,Arthur,D.
+83,Virginia,R.
+84,Christina,R.
+85,Theresa,M.
+86,Jason,C.
+87,Phillip,B.
+88,Adam,T.
+89,Margaret,J.
+90,Paul,P.
+91,Todd,W.
+92,Willie,O.
+93,Frances,R.
+94,Gregory,H.
+95,Lisa,P.
+96,Jacqueline,A.
+97,Shirley,D.
+98,Nicole,M.
+99,Mary,G.
+100,Jean,M.

seeds/raw_orders.csv

@@ -0,0 +1,100 @@
+id,user_id,order_date,status
+1,1,2018-01-01,returned
+2,3,2018-01-02,completed
+3,94,2018-01-04,completed
+4,50,2018-01-05,completed
+5,64,2018-01-05,completed
+6,54,2018-01-07,completed
+7,88,2018-01-09,completed
+8,2,2018-01-11,returned
+9,53,2018-01-12,completed
+10,7,2018-01-14,completed
+11,99,2018-01-14,completed
+12,59,2018-01-15,completed
+13,84,2018-01-17,completed
+14,40,2018-01-17,returned
+15,25,2018-01-17,completed
+16,39,2018-01-18,completed
+17,71,2018-01-18,completed
+18,64,2018-01-20,returned
+19,54,2018-01-22,completed
+20,20,2018-01-23,completed
+21,71,2018-01-23,completed
+22,86,2018-01-24,completed
+23,22,2018-01-26,return_pending
+24,3,2018-01-27,completed
+25,51,2018-01-28,completed
+26,32,2018-01-28,completed
+27,94,2018-01-29,completed
+28,8,2018-01-29,completed
+29,57,2018-01-31,completed
+30,69,2018-02-02,completed
+31,16,2018-02-02,completed
+32,28,2018-02-04,completed
+33,42,2018-02-04,completed
+34,38,2018-02-06,completed
+35,80,2018-02-08,completed
+36,85,2018-02-10,completed
+37,1,2018-02-10,completed
+38,51,2018-02-10,completed
+39,26,2018-02-11,completed
+40,33,2018-02-13,completed
+41,99,2018-02-14,completed
+42,92,2018-02-16,completed
+43,31,2018-02-17,completed
+44,66,2018-02-17,completed
+45,22,2018-02-17,completed
+46,6,2018-02-19,completed
+47,50,2018-02-20,completed
+48,27,2018-02-21,completed
+49,35,2018-02-21,completed
+50,51,2018-02-23,completed
+51,71,2018-02-24,completed
+52,54,2018-02-25,return_pending
+53,34,2018-02-26,completed
+54,54,2018-02-26,completed
+55,18,2018-02-27,completed
+56,79,2018-02-28,completed
+57,93,2018-03-01,completed
+58,22,2018-03-01,completed
+59,30,2018-03-02,completed
+60,12,2018-03-03,completed
+61,63,2018-03-03,completed
+62,57,2018-03-05,completed
+63,70,2018-03-06,completed
+64,13,2018-03-07,completed
+65,26,2018-03-08,completed
+66,36,2018-03-10,completed
+67,79,2018-03-11,completed
+68,53,2018-03-11,completed
+69,3,2018-03-11,completed
+70,8,2018-03-12,completed
+71,42,2018-03-12,shipped
+72,30,2018-03-14,shipped
+73,19,2018-03-16,completed
+74,9,2018-03-17,shipped
+75,69,2018-03-18,completed
+76,25,2018-03-20,completed
+77,35,2018-03-21,shipped
+78,90,2018-03-23,shipped
+79,52,2018-03-23,shipped
+80,11,2018-03-23,shipped
+81,76,2018-03-23,shipped
+82,46,2018-03-24,shipped
+83,54,2018-03-24,shipped
+84,70,2018-03-26,placed
+85,47,2018-03-26,shipped
+86,68,2018-03-26,placed
+87,46,2018-03-27,placed
+88,91,2018-03-27,shipped
+89,21,2018-03-28,placed
+90,66,2018-03-30,shipped
+91,47,2018-03-31,placed
+92,84,2018-04-02,placed
+93,66,2018-04-03,placed
+94,63,2018-04-03,placed
+95,27,2018-04-04,placed
+96,90,2018-04-06,placed
+97,89,2018-04-07,placed
+98,41,2018-04-07,placed
+99,85,2018-04-09,placed

seeds/raw_payments.csv

@@ -0,0 +1,114 @@
+id,order_id,payment_method,amount
+1,1,credit_card,1000
+2,2,credit_card,2000
+3,3,coupon,100
+4,4,coupon,2500
+5,5,bank_transfer,1700
+6,6,credit_card,600
+7,7,credit_card,1600
+8,8,credit_card,2300
+9,9,gift_card,2300
+10,9,bank_transfer,0
+11,10,bank_transfer,2600
+12,11,credit_card,2700
+13,12,credit_card,100
+14,13,credit_card,500
+15,13,bank_transfer,1400
+16,14,bank_transfer,300
+17,15,coupon,2200
+18,16,credit_card,1000
+19,17,bank_transfer,200
+20,18,credit_card,500
+21,18,credit_card,800
+22,19,gift_card,600
+23,20,bank_transfer,1500
+24,21,credit_card,1200
+25,22,bank_transfer,800
+26,23,gift_card,2300
+27,24,coupon,2600
+28,25,bank_transfer,2000
+29,25,credit_card,2200
+30,25,coupon,1600
+31,26,credit_card,3000
+32,27,credit_card,2300
+33,28,bank_transfer,1900
+34,29,bank_transfer,1200
+35,30,credit_card,1300
+36,31,credit_card,1200
+37,32,credit_card,300
+38,33,credit_card,2200
+39,34,bank_transfer,1500
+40,35,credit_card,2900
+41,36,bank_transfer,900
+42,37,credit_card,2300
+43,38,credit_card,1500
+44,39,bank_transfer,800
+45,40,credit_card,1400
+46,41,credit_card,1700
+47,42,coupon,1700
+48,43,gift_card,1800
+49,44,gift_card,1100
+50,45,bank_transfer,500
+51,46,bank_transfer,800
+52,47,credit_card,2200
+53,48,bank_transfer,300
+54,49,credit_card,600
+55,49,credit_card,900
+56,50,credit_card,2600
+57,51,credit_card,2900
+58,51,credit_card,100
+59,52,bank_transfer,1500
+60,53,credit_card,300
+61,54,credit_card,1800
+62,54,bank_transfer,1100
+63,55,credit_card,2900
+64,56,credit_card,400
+65,57,bank_transfer,200
+66,58,coupon,1800
+67,58,gift_card,600
+68,59,gift_card,2800
+69,60,credit_card,400
+70,61,bank_transfer,1600
+71,62,gift_card,1400
+72,63,credit_card,2900
+73,64,bank_transfer,2600
+74,65,credit_card,0
+75,66,credit_card,2800
+76,67,bank_transfer,400
+77,67,credit_card,1900
+78,68,credit_card,1600
+79,69,credit_card,1900
+80,70,credit_card,2600
+81,71,credit_card,500
+82,72,credit_card,2900
+83,73,bank_transfer,300
+84,74,credit_card,3000
+85,75,credit_card,1900
+86,76,coupon,200
+87,77,credit_card,0
+88,77,bank_transfer,1900
+89,78,bank_transfer,2600
+90,79,credit_card,1800
+91,79,credit_card,900
+92,80,gift_card,300
+93,81,coupon,200
+94,82,credit_card,800
+95,83,credit_card,100
+96,84,bank_transfer,2500
+97,85,bank_transfer,1700
+98,86,coupon,2300
+99,87,gift_card,3000
+100,87,credit_card,2600
+101,88,credit_card,2900
+102,89,bank_transfer,2200
+103,90,bank_transfer,200
+104,91,credit_card,1900
+105,92,bank_transfer,1500
+106,92,coupon,200
+107,93,gift_card,2600
+108,94,coupon,700
+109,95,coupon,2400
+110,96,gift_card,1700
+111,97,bank_transfer,1400
+112,98,bank_transfer,1000
+113,99,credit_card,2400