Merge pull request #541 from m29h/issue_468

Remove MySQL and sqlite dependencies, resolves #468

.github/workflows/go.yml

@@ -27,10 +27,6 @@ jobs:
         go-version: stable
       id: go
 
-    - name: Setup
-      if: runner.os == 'Linux'
-      run: docker-compose up -d mysql
-
     - name: Generate go test Slice
       id: test_split
       uses: hashicorp-forge/go-test-split-action@v1

.github/workflows/release.yml

@@ -21,10 +21,6 @@ jobs:
     - name: Check out code into the Go module directory
       uses: actions/checkout@v3
 
-    - name: Setup
-      if: runner.os == 'Linux'
-      run: docker-compose up -d mysql
-
     - name: Short Tests
       if: runner.os == 'Linux'
       env:

Dockerfile

@@ -1,22 +1,18 @@
-FROM golang:1.21-bookworm as backend
-WORKDIR /go/src/github.com/moov-io/watchman
-RUN apt-get update && apt-get upgrade -y && apt-get install make gcc g++
-COPY . .
+FROM golang:alpine as backend
+WORKDIR /src
+COPY . /src
 RUN go mod download
-RUN make build-server
+RUN CGO_ENABLED=0 go build -o ./bin/server /src/cmd/server
 
-FROM node:21-bookworm as frontend
+FROM node:21-alpine as frontend
 COPY webui/ /watchman/
 WORKDIR /watchman/
 RUN npm install --legacy-peer-deps
 RUN npm run build
 
-FROM debian:stable-slim
+FROM alpine:latest
 LABEL maintainer="Moov <oss@moov.io>"
-
-RUN apt-get update && apt-get upgrade -y && apt-get install -y ca-certificates
-COPY --from=backend /go/src/github.com/moov-io/watchman/bin/server /bin/server
-
+COPY --from=backend /src/bin/server /bin/server
 COPY --from=frontend /watchman/build/ /watchman/
 ENV WEB_ROOT=/watchman/
 

README.md

@@ -199,15 +199,13 @@ PONG
 | `UNMATCHED_INDEX_TOKEN_WEIGHT` | Weight of penalty applied to scores when part of the indexed name isn't matched. | 0.15    |
 | `JARO_WINKLER_BOOST_THRESHOLD` | Jaro-Winkler boost threshold. | 0.7 |
 | `JARO_WINKLER_PREFIX_SIZE` | Jaro-Winkler prefix size. | 4 |
-| `WEBHOOK_BATCH_SIZE` | How many watches to read from database per batch of async searches. | 100 |
 | `LOG_FORMAT` | Format for logging lines to be written as. | Options: `json`, `plain` - Default: `plain` |
 | `LOG_LEVEL` | Level of logging to emit. | Options: `trace`, `info` - Default: `info` |
 | `BASE_PATH` | HTTP path to serve API and web UI from. | `/` |
 | `HTTP_BIND_ADDRESS` | Address to bind HTTP server on. This overrides the command-line flag `-http.addr`. | Default: `:8084` |
 | `HTTP_ADMIN_BIND_ADDRESS` | Address to bind admin HTTP server on. This overrides the command-line flag `-admin.addr`. | Default: `:9094` |
 | `HTTPS_CERT_FILE` | Filepath containing a certificate (or intermediate chain) to be served by the HTTP server. Requires all traffic be over secure HTTP. | Empty |
 | `HTTPS_KEY_FILE`  | Filepath of a private key matching the leaf certificate from `HTTPS_CERT_FILE`. | Empty |
-| `DATABASE_TYPE` | Which database option to use (Options: `sqlite`, `mysql`). | Default: `sqlite` |
 | `WEB_ROOT` | Directory to serve web UI from. | Default: `webui/` |
 | `WEBHOOK_MAX_WORKERS` | Maximum number of workers processing webhooks. | Default: 10 |
 | `DOWNLOAD_WEBHOOK_URL` | Optional webhook URL called when data downloads / refreshes occur. | Empty |
@@ -228,27 +226,6 @@ PONG
 | `KEEP_STOPWORDS` | Boolean to keep stopwords in names. | `false` |
 | `DEBUG_NAME_PIPELINE` | Boolean to print debug messages for each name (SDN, SSI) processing step. | `false` |
 
-#### Storage
-
-Based on `DATABASE_TYPE`, the following environmental variables will be read to configure connections for a specific database.
-
-##### MySQL
-
-- `MYSQL_ADDRESS`: TCP address for connecting to the MySQL server. (example: `tcp(hostname:3306)`)
-- `MYSQL_DATABASE`: Name of database to connect into.
-- `MYSQL_PASSWORD`: Password of user account for authentication.
-- `MYSQL_USER`: Username used for authentication.
-
-Refer to the MySQL driver documentation for [connection parameters](https://github.com/go-sql-driver/mysql#dsn-data-source-name).
-
-- `MYSQL_TIMEOUT`: Timeout parameter specified on (DSN) data source name. (Default: `30s`)
-
-##### SQLite
-
-- `SQLITE_DB_PATH`: Local filepath location for the SQLite database. (Default: `watchman.db`)
-
-Refer to the SQLite driver documentation for [connection parameters](https://github.com/mattn/go-sqlite3#connection-string).
-
 ##### Downloads
 
 Moov Watchman supports sending a webhook (`POST` HTTP Request) when the underlying data is refreshed. The body will be the count of entities indexed for each list. The body will be in JSON format and the same schema as the manual data refresh endpoint.
@@ -268,8 +245,6 @@ Moov Watchman supports sending a webhook periodically with a free-form name of a
 - `last_data_refresh_count`: Count of records for a given sanction or entity list.
 - `match_percentages` A histogram which holds the match percentages with a label (`type`) of searches.
    - `type`: Can be address, q, remarksID, name, altName
-- `mysql_connections`: Number of MySQL connections and their statuses.
-- `sqlite_connections`: Number of SQLite connections and their statuses.
 
 ### Data persistence
 

cmd/server/download.go

@@ -6,22 +6,18 @@ package main
 
 import (
 	"bytes"
-	"database/sql"
 	"encoding/json"
 	"errors"
 	"fmt"
-	"net/http"
 	"os"
 	"strings"
 	"time"
 
-	moovhttp "github.com/moov-io/base/http"
 	"github.com/moov-io/base/log"
 	"github.com/moov-io/watchman/pkg/csl"
 	"github.com/moov-io/watchman/pkg/dpl"
 	"github.com/moov-io/watchman/pkg/ofac"
 
-	"github.com/gorilla/mux"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -110,7 +106,7 @@ func (ss *DownloadStats) MarshalJSON() ([]byte, error) {
 
 // periodicDataRefresh will forever block for interval's duration and then download and reparse the data.
 // Download stats are recorded as part of a successful re-download and parse.
-func (s *searcher) periodicDataRefresh(interval time.Duration, downloadRepo downloadRepository, updates chan *DownloadStats) {
+func (s *searcher) periodicDataRefresh(interval time.Duration, updates chan *DownloadStats) {
 	if interval == 0*time.Second {
 		s.logger.Logf("not scheduling periodic refreshing duration=%v", interval)
 		return
@@ -123,7 +119,6 @@ func (s *searcher) periodicDataRefresh(interval time.Duration, downloadRepo down
 				s.logger.Info().Logf("ERROR: refreshing data: %v", err)
 			}
 		} else {
-			downloadRepo.recordStats(stats)
 			if s.logger != nil {
 				s.logger.Info().With(log.Fields{
 					// OFAC
@@ -450,85 +445,3 @@ func lastRefresh(dir string) time.Time {
 	}
 	return oldest.In(time.UTC)
 }
-
-func addDownloadRoutes(logger log.Logger, r *mux.Router, repo downloadRepository) {
-	r.Methods("GET").Path("/downloads").HandlerFunc(getLatestDownloads(logger, repo))
-}
-
-func getLatestDownloads(logger log.Logger, repo downloadRepository) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		w = wrapResponseWriter(logger, w, r)
-
-		limit := extractSearchLimit(r)
-		downloads, err := repo.latestDownloads(limit)
-		if err != nil {
-			moovhttp.Problem(w, err)
-			return
-		}
-
-		logger.Info().With(log.Fields{
-			"requestID": log.String(moovhttp.GetRequestID(r)),
-		}).Log("get latest downloads")
-
-		w.Header().Set("Content-Type", "application/json; charset=utf-8")
-		w.WriteHeader(http.StatusOK)
-		if err := json.NewEncoder(w).Encode(downloads); err != nil {
-			moovhttp.Problem(w, err)
-			return
-		}
-	}
-}
-
-type downloadRepository interface {
-	latestDownloads(limit int) ([]DownloadStats, error)
-	recordStats(stats *DownloadStats) error
-}
-
-type sqliteDownloadRepository struct {
-	db     *sql.DB
-	logger log.Logger
-}
-
-func (r *sqliteDownloadRepository) close() error {
-	return r.db.Close()
-}
-
-func (r *sqliteDownloadRepository) recordStats(stats *DownloadStats) error {
-	if stats == nil {
-		return errors.New("recordStats: nil downloadStats")
-	}
-
-	query := `insert into download_stats (downloaded_at, sdns, alt_names, addresses, sectoral_sanctions, denied_persons, bis_entities) values (?, ?, ?, ?, ?, ?, ?);`
-	stmt, err := r.db.Prepare(query)
-	if err != nil {
-		return err
-	}
-	defer stmt.Close()
-
-	_, err = stmt.Exec(stats.RefreshedAt, stats.SDNs, stats.Alts, stats.Addresses, stats.SectoralSanctions, stats.DeniedPersons, stats.BISEntities)
-	return err
-}
-
-func (r *sqliteDownloadRepository) latestDownloads(limit int) ([]DownloadStats, error) {
-	query := `select downloaded_at, sdns, alt_names, addresses, sectoral_sanctions, denied_persons, bis_entities from download_stats order by downloaded_at desc limit ?;`
-	stmt, err := r.db.Prepare(query)
-	if err != nil {
-		return nil, err
-	}
-	defer stmt.Close()
-
-	rows, err := stmt.Query(limit)
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-
-	var downloads []DownloadStats
-	for rows.Next() {
-		var dl DownloadStats
-		if err := rows.Scan(&dl.RefreshedAt, &dl.SDNs, &dl.Alts, &dl.Addresses, &dl.SectoralSanctions, &dl.DeniedPersons, &dl.BISEntities); err == nil {
-			downloads = append(downloads, dl)
-		}
-	}
-	return downloads, rows.Err()
-}

cmd/server/download_handler.go

@@ -9,7 +9,6 @@ import (
 	"net/http"
 	"time"
 
-	moovhttp "github.com/moov-io/base/http"
 	"github.com/moov-io/base/log"
 )
 
@@ -18,18 +17,14 @@ const (
 )
 
 // manualRefreshHandler will register an endpoint on the admin server data refresh endpoint
-func manualRefreshHandler(logger log.Logger, searcher *searcher, updates chan *DownloadStats, downloadRepo downloadRepository) http.HandlerFunc {
+func manualRefreshHandler(logger log.Logger, searcher *searcher, updates chan *DownloadStats) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		logger.Log("admin: refreshing data")
 
 		if stats, err := searcher.refreshData(""); err != nil {
 			logger.LogErrorf("ERROR: admin: problem refreshing data: %v", err)
 			w.WriteHeader(http.StatusInternalServerError)
 		} else {
-			if err := downloadRepo.recordStats(stats); err != nil {
-				moovhttp.Problem(w, err)
-				return
-			}
 
 			go func() {
 				updates <- stats