add database for crisp conversation segments

scripts/insert_crisp_conversation_segments.sql

@@ -0,0 +1,31 @@
+
+
+/* Crisp conversation segments
+*/
+CREATE TABLE IF NOT EXISTS crisp_conversation_segments (
+  session_id VARCHAR NOT NULL,
+  state VARCHAR NOT NULL,
+  segment VARCHAR,
+  created_at timestamp NOT NULL,
+  updated_at timestamp NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS crisp_conversation_segments_idx ON crisp_conversation_segments (session_id);
+CREATE INDEX IF NOT EXISTS crisp_conversation_segment_idx ON crisp_conversation_segments (segment);
+CREATE INDEX IF NOT EXISTS crisp_conversation_state_idx ON crisp_conversation_segments (state);
+CREATE INDEX IF NOT EXISTS crisp_created_at_idx ON crisp_conversation_segments (created_at);
+
+/* Insert data from the csv file into the DB. */
+/* If data is already present, update the content field */
+
+CREATE TEMPORARY TABLE crisp_conversation_segments_temp (session_id VARCHAR, state VARCHAR, segment VARCHAR, created_at timestamp, updated_at timestamp);
+
+-- CSV file has fields in this order : 
+-- If it changes, change this line or it will break.
+\copy crisp_conversation_segments_temp(session_id, state, segment, created_at, updated_at) FROM '/app/crisp_conversation_segments.csv' DELIMITER ',' CSV HEADER;
+
+
+INSERT INTO crisp_conversation_segments (session_id, state, segment, created_at, updated_at)
+SELECT *
+FROM crisp_conversation_segments_temp
+ON CONFLICT DO NOTHING

sync_data.sh

@@ -27,6 +27,9 @@ echo "Fetch S3 pushers $extract_date"
 time ./fetch_from_s3.sh pushers $extract_date
 echo "Fetch S3 account_data $extract_date"
 time ./fetch_from_s3.sh account_data $extract_date
+echo "Fetch S3 crisp_conversation_segments $extract_date"
+time ./fetch_from_s3.sh crisp_conversation_segments $extract_date
+
 
 ## Set up DB
 psql -d $DATABASE_URL -f scripts/tables.sql
@@ -42,6 +45,8 @@ echo "Insert Pushers"
 time psql -d $DATABASE_URL -f scripts/insert_pushers_data.sql
 echo "Insert Account Data"
 time psql -d $DATABASE_URL -f scripts/insert_account_data_data.sql
+echo "Insert crisp conversation segments"
+time psql -d $DATABASE_URL -f scripts/insert_crisp_conversation_segments.sql
 
 echo "Done !"