From 6152cf7fc63efbbfc7777160570b0562ecf01e22 Mon Sep 17 00:00:00 2001 From: Sean Fischer Date: Thu, 15 Apr 2021 18:37:09 -0400 Subject: [PATCH] Refactor to un-export some vars --- .gitignore | 3 +++ internal/database/connect.go | 14 +++++++------- internal/database/etl.go | 24 ++++++++++++------------ internal/database/models.go | 21 +++++++-------------- sql/bootstrap.sql | 9 +++++++++ sql/create_counts.sql | 13 +++++++++++++ sql/create_symbols.sql | 9 +++++++++ 7 files changed, 60 insertions(+), 33 deletions(-) create mode 100644 sql/bootstrap.sql create mode 100644 sql/create_counts.sql create mode 100644 sql/create_symbols.sql diff --git a/.gitignore b/.gitignore index 3cbc843..778edce 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +*.db +.dolt + resources/ *.ignore */dist/* diff --git a/internal/database/connect.go b/internal/database/connect.go index 8087529..c4fab4d 100644 --- a/internal/database/connect.go +++ b/internal/database/connect.go @@ -14,20 +14,20 @@ const ( func Connect(input ConnectionInput) (conn Connection, err error) { - conn.EtlResultsTable = os.Getenv("ETL_RESULTS_TABLE") - conn.RedditPostArchiveBucket = os.Getenv("REDDIT_ARCHIVE_BUCKET") + conn.etlResultsTable = os.Getenv("ETL_RESULTS_TABLE") + conn.redditPostArchiveBucket = os.Getenv("REDDIT_ARCHIVE_BUCKET") - if conn.EtlResultsTable == "" { + if conn.etlResultsTable == "" { return conn, fmt.Errorf("Could not find required table name ETL_RESULTS_TABLE") } - if conn.RedditPostArchiveBucket == "" { + if conn.redditPostArchiveBucket == "" { return conn, fmt.Errorf("Could not find required table name REDDIT_ARCHIVE_BUCKET") } - conn.Session = input.Session - conn.Service = dynamodb.New(conn.Session) - conn.S3Service = s3.New(conn.Session) + conn.session = input.Session + conn.service = dynamodb.New(conn.session) + conn.s3Service = s3.New(conn.session) return conn, err } diff --git a/internal/database/etl.go b/internal/database/etl.go index a77e0cc..8996d1c 100644 --- a/internal/database/etl.go +++ b/internal/database/etl.go @@ -44,11 +44,11 @@ func (c *Connection) PutRedditPostArchiveRecord(record RedditPostArchiveRecord) } input := &s3.PutObjectInput{ - Bucket: aws.String(c.RedditPostArchiveBucket), + Bucket: aws.String(c.redditPostArchiveBucket), Key: aws.String(record.Key), Body: bytes.NewReader(buf), } - _, err = c.S3Service.PutObject(input) + _, err = c.s3Service.PutObject(input) return err } @@ -62,11 +62,11 @@ func (c *Connection) ListRedditPostArchiveRecord(input RedditPostArchiveListInpu ) listInput := &s3.ListObjectsV2Input{ - Bucket: aws.String(c.RedditPostArchiveBucket), + Bucket: aws.String(c.redditPostArchiveBucket), Prefix: aws.String(startAfterPrefix), } - objectList, err := c.S3Service.ListObjectsV2(listInput) + objectList, err := c.s3Service.ListObjectsV2(listInput) if err != nil { return keys, err } @@ -81,11 +81,11 @@ func (c *Connection) ListRedditPostArchiveRecord(input RedditPostArchiveListInpu func (c *Connection) GetRedditPostArchiveRecord(input RedditPostArchiveQueryInput) (record RedditPostArchiveRecord, err error) { getObjectInput := &s3.GetObjectInput{ - Bucket: aws.String(c.RedditPostArchiveBucket), + Bucket: aws.String(c.redditPostArchiveBucket), Key: aws.String(input.Key), } - object, err := c.S3Service.GetObject(getObjectInput) + object, err := c.s3Service.GetObject(getObjectInput) if err != nil { return record, err } @@ -122,10 +122,10 @@ func (c *Connection) PutEtlResultsRecord(record EtlResultsRecord) (err error) { input := &dynamodb.PutItemInput{ Item: av, - TableName: aws.String(c.EtlResultsTable), + TableName: aws.String(c.etlResultsTable), } - _, err = c.Service.PutItem(input) + _, err = c.service.PutItem(input) return err } @@ -139,10 +139,10 @@ func (c *Connection) GetEtlResultsRecord(input EtlResultsQueryInput) (record []E }, }, KeyConditionExpression: aws.String("id = :v1"), - TableName: aws.String(c.EtlResultsTable), + TableName: aws.String(c.etlResultsTable), } - result, err := c.Service.Query(qInput) + result, err := c.service.Query(qInput) if err != nil { return record, err @@ -179,10 +179,10 @@ func (c *Connection) GetLatestEtlResultsRecord(input EtlResultsQueryInput) (reco KeyConditionExpression: aws.String("id = :v1"), ScanIndexForward: aws.Bool(false), Limit: aws.Int64(1), - TableName: aws.String(c.EtlResultsTable), + TableName: aws.String(c.etlResultsTable), } - result, err = c.Service.Query(qInput) + result, err = c.service.Query(qInput) if err != nil { return record, err } diff --git a/internal/database/models.go b/internal/database/models.go index 4df5903..f481dd1 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -12,24 +12,17 @@ import ( ) type Connection struct { - // Session is the shared aws session - Session *session.Session + // session is the shared aws session + session *session.Session // Service is the shared dynamo db connection - Service *dynamodb.DynamoDB - - // S3Service is the shared S3 connection - S3Service *s3.S3 - - EtlResultsTable string - RedditPostArchiveBucket string - ApiKeyTable string + service *dynamodb.DynamoDB - // YahooTrendingTable is deprecated - YahooTrendingTable string + // s3Service is the shared S3 connection + s3Service *s3.S3 - // UserTable is deprecated - UserTable string + etlResultsTable string + redditPostArchiveBucket string } type EtlResultsRecord struct { diff --git a/sql/bootstrap.sql b/sql/bootstrap.sql new file mode 100644 index 0000000..8f6b7d0 --- /dev/null +++ b/sql/bootstrap.sql @@ -0,0 +1,9 @@ +-- Create the main datbase +CREATE DATABASE IF NOT EXISTS kettle; +USE kettle; + + + +-- Create insert query + +-- Create update symbols query diff --git a/sql/create_counts.sql b/sql/create_counts.sql new file mode 100644 index 0000000..0cfc41d --- /dev/null +++ b/sql/create_counts.sql @@ -0,0 +1,13 @@ +-- Create count table +CREATE TABLE IF NOT EXISTS Counts ( + Id varchar(25), + CountDate datetime, + Ticker varchar(10), + PostScore int, + CommentScore int, + TotalScore float, + PostMentions int, + CommentMentions int, + PRIMARY KEY(Id) +); + diff --git a/sql/create_symbols.sql b/sql/create_symbols.sql new file mode 100644 index 0000000..e37dbeb --- /dev/null +++ b/sql/create_symbols.sql @@ -0,0 +1,9 @@ +-- Create ticker symbol table +CREATE TABLE IF NOT EXISTS Symbols ( + Ticker varchar(10), + Exchange varchar(255), + FullName varchar(255), + ETF boolean, + PRIMARY KEY(Ticker) +); +