From 60b1033dba2633e6bb50656780239129763f145c Mon Sep 17 00:00:00 2001 From: ysandler Date: Sun, 27 Apr 2025 14:29:20 -0500 Subject: [PATCH] feat: created simple api for client testing --- .env.example | 1 + README.md | 121 +++++++++++++++++++++++------- main.go | 102 ++++++++++++++++++++++---- q/dto.go | 187 +++++++++++++++++++++++++++++++++++++++++++++++ q/query.go | 18 ----- q/select.go | 105 +------------------------- q/select_test.go | 35 +++++++++ 7 files changed, 410 insertions(+), 159 deletions(-) create mode 100644 .env.example create mode 100644 q/dto.go diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..8a8e175 --- /dev/null +++ b/.env.example @@ -0,0 +1 @@ +PORT=6796 diff --git a/README.md b/README.md index 7b40ad1..7d96911 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,68 @@ Core program to interpret query language strings into structured data, and back again. +## How to Use the Project + +### Microservice for client applications + +At least for now, this can be treated like a micro service. Very simply you query the endpoint(s) +with your SQL strings and retrieve the structured data back. + +``` +POST /query +body: { + sql: string +} +``` + +Be aware that the api is not currently converting enums to their string representations, so you might expect +JoinType to be `"INNER"`, bit it is returned as `0`. Refer to the [dto](q/dto.go) as a reference to what the +`string` values for these `enum`s would be. + +> `iota` is incrementing for all values in the `enum`, and starts at `0` unless modified. `iota + 1` will start at `1` + +Right now we are only parsing SELECT statements. If you try to do something else it will either error out +or hang. The HTTP response should timeout after 30 seconds. + +### Development on core logic + +This project is wored on via TDD methods, it is the only way to do so as the parsing of SQL is so janky. If +you are wanting to add a feature to the parsing, you need to first write a unit test. + +Become familiar with [select_test](q/select_test.md) to see how we are doing it. In Brief: + +We have the test struct where `input` is the entire SQL string you are testing, and expected is +the exact struct (of which ever query struct) you expect to see returned. +```go +type ParsingTest struct { + input string + expected Select +} +``` + +Add yours to the `var testSqlStatements []ParsingTest` of the file. + +If you are adding a new field to the query's struct, or modifying any fields, make sure to add or update the +conditionals in teh `t.Run(testName, func(t *testing.T)` block. + +#### Remember the TDD Process +- Write enough of a test to make sure it fails +- Write enough prod code to make sure it passes +- Repeat until finished developing the feature + +### Starting the app + +**Prerequisites:** + +* Go installed (version X.Y or higher - check your code for specifics) +* `go mod tidy` to fetch dependencies +* `cp .env.example .env` to create your own .env file + +**Running the App:** + +1. `go run main.go` to start the server (PORT is determined by the .env file) +2. `go test ./q` to run test suite if developing features (add `-v` if you want a verbose output) + ## Data Structure Philosophy We are operating off of the philosophy that the first class data is SQL Statement stings. @@ -68,19 +130,43 @@ already been processed through the scan. Making naming and reading new flags tri A `Select` object is shaped as the following: ```go type Select struct { - Table string - Columns []Column - Conditionals []Conditional - OrderBys []OrderBy - Joins []Join - IsWildcard bool - IsDistinct bool + Table string `json:"table"` + Columns []Column `json:"columns"` + Conditionals []Conditional `json:"conditionals"` + OrderBys []OrderBy `json:"order_bys"` + Joins []Join `json:"joins"` + IsWildcard bool `json:"is_wildcard"` + IsDistinct bool `json:"is_distinct"` } type Column struct { - Name string - Alias string - AggregateFunction AggregateFunctionType + Name string `json:"name"` + Alias string `json:"alias"` + AggregateFunction AggregateFunctionType `json:"aggregate_function"` // Changed type name to match Go naming conventions +} + +type Conditional struct { + Key string `json:"key"` + Operator string `json:"operator"` + Value string `json:"value"` + Extension string `json:"extension"` +} + +type OrderBy struct { + Key string + IsDescend bool // SQL queries with no ASC|DESC on their ORDER BY are ASC by default, hence why this bool for the opposite +} + +type Join struct { + Type JoinType `json:"type"` + Table Table `json:"table"` + Ons []Conditional `json:"ons"` +} + +// Only used in Join.Table right now, but Select.Table will also use this soon +type Table struct { + Name string `json:"name"` + Alias string `json:"alias"` } type AggregateFunctionType int @@ -92,23 +178,8 @@ const ( AVG ) -//dependency in query.go -type Conditional struct { - Key string - Operator string - Value string - DataType string - Extension string // AND, OR, etc -} - - -type OrderBy struct { - Key string - IsDescend bool // SQL queries with no ASC|DESC on their ORDER BY are ASC by default, hence why this bool for the opposite -} ``` - ## Improvement Possibilities - Maybe utilize the `lookBehindBuffer` more to cut down the number of state flags in the scans? diff --git a/main.go b/main.go index d655fe3..cf039a0 100644 --- a/main.go +++ b/main.go @@ -1,28 +1,41 @@ package main import ( + "context" + "encoding/json" "fmt" - //"query-inter/q" + "io" + "os" - "github.com/DataDog/go-sqllexer" - // "github.com/DataDog/go-sqllexer" + //"github.com/DataDog/go-sqllexer" + "log" + "net/http" + "query-inter/q" + "time" ) func main() { - selectQuery := "SELECT MIN(Price) AS SmallestPrice, CategoryID FROM Products GROUP BY CategoryID;" + //selectQuery := "SELECT MIN(Price) AS SmallestPrice, CategoryID FROM Products GROUP BY CategoryID;" + PORT := os.Getenv("PORT") + + if PORT == "" { + log.Panicln("PORT could not be read from env") + } + + StartServer(PORT) //allStatements := q.ExtractSqlStatmentsFromString(selectQuery) //fmt.Println(allStatements) - lexer := sqllexer.New(selectQuery) - for { - token := lexer.Scan() - fmt.Println(token.Value, token.Type) - - if token.Type == sqllexer.EOF { - break - } - } + //lexer := sqllexer.New(selectQuery) + //for { + // token := lexer.Scan() + // fmt.Println(token.Value, token.Type) + // + // if token.Type == sqllexer.EOF { + // break + // } + //} //for _, sql := range allStatements { //query := q.ParseSelectStatement(sql) @@ -32,3 +45,66 @@ func main() { //} } + +func StartServer(port string) { + http.HandleFunc("/query", HandlePostQuery) + fmt.Println("Starting Server on 8080") + fmt.Println("call POST /query with { sql: string }") + log.Fatal(http.ListenAndServe(":"+port, nil)) +} + +type QueryRequest struct { + SQL string `json:"sql"` +} + +func HandlePostQuery(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "Error reading request body", http.StatusBadRequest) + return + } + defer r.Body.Close() + + var request QueryRequest + err = json.Unmarshal(body, &request) + if err != nil { + http.Error(w, "Error parsing JSON: "+string(body), http.StatusBadRequest) + return + } + + ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second) + defer cancel() + + var query q.Select + done := make(chan struct{}) + + go func() { + defer cancel() + query = q.ParseSelectStatement(request.SQL) + done <- struct{}{} + }() + + select { + case <-done: + // Proceed + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) // Set the HTTP status code + + jsonData, err := q.MarshalSelect(query) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + _, err = w.Write(jsonData) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + case <-ctx.Done(): + http.Error(w, "Request timed out", http.StatusRequestTimeout) + return + } + +} diff --git a/q/dto.go b/q/dto.go new file mode 100644 index 0000000..cffa7d4 --- /dev/null +++ b/q/dto.go @@ -0,0 +1,187 @@ +package q + +import ( + "encoding/json" + "strings" +) + +const NONE = 0 + +type QueryType int + +const ( + SELECT QueryType = iota + 1 + UPDATE + INSERT + DELETE +) + +// QueryTypeString converts a QueryType integer to its string representation. +func QueryTypeString(t QueryType) string { + switch t { + case SELECT: + return "SELECT" + case UPDATE: + return "UPDATE" + case INSERT: + return "INSERT" + case DELETE: + return "DELETE" + default: + return "UNKNOWN" + } +} + +type Table struct { + Name string `json:"name"` + Alias string `json:"alias"` +} + +type Conditional struct { + Key string `json:"key"` + Operator string `json:"operator"` + Value string `json:"value"` + Extension string `json:"extension"` +} + +type AggregateFunctionType int + +const ( + MIN AggregateFunctionType = iota + 1 + MAX + COUNT + SUM + AVG +) + +// AggregateFunctionTypeString converts an AggregateFunctionType integer to its string representation. +func AggregateFunctionTypeString(t AggregateFunctionType) string { + switch t { + case MIN: + return "MIN" + case MAX: + return "MAX" + case COUNT: + return "COUNT" + case SUM: + return "SUM" + case AVG: + return "AVG" + default: + return "UNKNOWN" + } +} + +// Converts the string name of AggregateFunctionType into it original int +func AggregateFunctionTypeByName(name string) AggregateFunctionType { + var functionType AggregateFunctionType + switch strings.ToUpper(name) { + case "MIN": + functionType = MIN + case "MAX": + functionType = MAX + case "COUNT": + functionType = COUNT + case "SUM": + functionType = SUM + case "AVG": + functionType = AVG + default: + functionType = 0 + } + + return functionType +} + +type JoinType int + +const ( + INNER JoinType = iota + LEFT + RIGHT + FULL + SELF +) + +// JoinTypeString converts a JoinType integer to its string representation. +func JoinTypeString(t JoinType) string { + switch t { + case INNER: + return "INNER" + case LEFT: + return "LEFT" + case RIGHT: + return "RIGHT" + case FULL: + return "FULL" + case SELF: + return "SELF" + default: + return "UNKNOWN" + } +} + +type Column struct { + Name string `json:"name"` + Alias string `json:"alias"` + AggregateFunction AggregateFunctionType `json:"aggregate_function"` // Changed type name to match Go naming conventions +} + +type Join struct { + Type JoinType `json:"type"` + Table Table `json:"table"` + Ons []Conditional `json:"ons"` +} + +type Select struct { + Table string `json:"table"` + Columns []Column `json:"columns"` + Conditionals []Conditional `json:"conditionals"` + OrderBys []OrderBy `json:"order_bys"` + Joins []Join `json:"joins"` + IsWildcard bool `json:"is_wildcard"` + IsDistinct bool `json:"is_distinct"` +} + +type OrderBy struct { + Key string `json:"key"` + IsDescend bool `json:"is_descend"` +} + +func MarshalSelect(selectStatement Select) ([]byte, error) { + jsonData, err := json.MarshalIndent(selectStatement, "", " ") + return jsonData, err +} + +//func test() { +//selectStatement := Select{ +// Table: "users", +// Columns: []Column{ +// {Name: "id", Alias: "user_id", AggregateFunction: COUNT}, +// }, +// Conditionals: []Conditional{ +// {Key: "age", Operator: ">", Value: "18"}, +// }, +// OrderBys: []OrderBy{ +// {Key: "name", IsDescend: false}, +// }, +// Joins: []Join{ +// { +// Type: LEFT, +// Table: Table{Name: "orders"}, +// Ons: []Conditional{{Key: "user_id", Operator: "=", Value: "users.id"}}, +// }}, +//} +// +//jsonData, err := json.MarshalIndent(selectStatement, "", " ") +//if err != nil { +// fmt.Println("Error marshaling JSON:", err) +// return +//} +// +//fmt.Println(string(jsonData)) +// +//fmt.Println(QueryTypeString(SELECT)) +//fmt.Println(AggregateFunctionTypeString(MAX)) +//fmt.Println(JoinTypeString(INNER)) +//} diff --git a/q/query.go b/q/query.go index 27ea0c8..d8350ad 100644 --- a/q/query.go +++ b/q/query.go @@ -10,24 +10,6 @@ type Query interface { GetFullSql() string } -const NONE = 0 - -type QueryType int - -const ( - SELECT QueryType = iota + 1 - UPDATE - INSERT - DELETE -) - -type Conditional struct { - Key string - Operator string - Value string - Extension string // AND, OR, etc -} - func GetQueryTypeFromToken(token *sqllexer.Token) QueryType { if token.Type != sqllexer.COMMAND { return NONE diff --git a/q/select.go b/q/select.go index baf2085..eb1e249 100644 --- a/q/select.go +++ b/q/select.go @@ -7,106 +7,13 @@ import ( "github.com/DataDog/go-sqllexer" ) -type AggregateFunctionType int - -const ( - MIN AggregateFunctionType = iota + 1 - MAX - COUNT - SUM - AVG -) - -type JoinType int - -const ( - INNER JoinType = iota - LEFT - RIGHT - FULL - SELF -) - -type Table struct { - Name string - Alias string -} - -type Column struct { - Name string - Alias string - AggregateFunction AggregateFunctionType -} - -type Join struct { - Type JoinType - Table Table - Ons []Conditional -} - -type Select struct { - Table string - Columns []Column - Conditionals []Conditional - OrderBys []OrderBy - Joins []Join - IsWildcard bool - IsDistinct bool -} - -type OrderBy struct { - Key string - IsDescend bool // SQL queries with no ASC|DESC on their ORDER BY are ASC by default, hence why this bool for the opposite -} - -func GetAggregateFunctionTypeByName(name string) AggregateFunctionType { - var functionType AggregateFunctionType - switch strings.ToUpper(name) { - case "MIN": - functionType = MIN - case "MAX": - functionType = MAX - case "COUNT": - functionType = COUNT - case "SUM": - functionType = SUM - case "AVG": - functionType = AVG - default: - functionType = 0 - } - - return functionType -} - -func GetAggregateFunctionNameByType(functionType AggregateFunctionType) string { - var functionName string - switch functionType { - case MIN: - functionName = "MIN" - case MAX: - functionName = "MAX" - case COUNT: - functionName = "COUNT" - case SUM: - functionName = "SUM" - case AVG: - functionName = "AVG" - default: - functionName = "" - } - - return functionName - -} - func GetFullStringFromColumn(column Column) string { var workingSlice string if column.AggregateFunction > 0 { workingSlice = fmt.Sprintf( "%s(%s)", - GetAggregateFunctionNameByType(column.AggregateFunction), + AggregateFunctionTypeString(column.AggregateFunction), column.Name, ) } else { @@ -140,7 +47,6 @@ func (q *Select) GetFullSql() string { workingSqlSlice = append(workingSqlSlice, "FROM "+q.Table) - // TODO: need to account for `AND` and `OR`s and stuff for _, condition := range q.Conditionals { workingSqlSlice = append(workingSqlSlice, condition.Key) workingSqlSlice = append(workingSqlSlice, condition.Operator) @@ -152,13 +58,6 @@ func (q *Select) GetFullSql() string { return fullSql } -func mutateSelectFromKeyword(query *Select, keyword string) { - switch strings.ToUpper(keyword) { - case "DISTINCT": - query.IsDistinct = true - } -} - func unshiftBuffer(buf *[10]sqllexer.Token, value sqllexer.Token) { for i := 9; i >= 1; i-- { buf[i] = buf[i-1] @@ -215,7 +114,7 @@ func ParseSelectStatement(sql string) Select { continue } else if token.Type == sqllexer.FUNCTION { unshiftBuffer(&lookBehindBuffer, *token) - workingColumn.AggregateFunction = GetAggregateFunctionTypeByName(token.Value) + workingColumn.AggregateFunction = AggregateFunctionTypeByName(token.Value) continue } else if token.Type == sqllexer.PUNCTUATION { if token.Value == "," { diff --git a/q/select_test.go b/q/select_test.go index cd20c2f..555e516 100644 --- a/q/select_test.go +++ b/q/select_test.go @@ -137,6 +137,32 @@ func TestParseSelectStatement(t *testing.T) { }, }, }, + // { + // input: "SELECT ProductID, ProductName, CategoryName FROM Products INNER JOIN Categories ON Products.CategoryID = Categories.CategoryID; ", + // expected: Select{ + // Table: "Products", + // Columns: []Column{ + // {Name: "ProductID"}, + // {Name: "ProductName"}, + // {Name: "CategoryName"}, + // }, + // Joins: []Join{ + // { + // Type: INNER, + // Table: Table{ + // Name: "Categories", + // }, + // Ons: []Conditional{ + // { + // Key: "Products.CategoryID", + // Operator: "=", + // Value: "Categories.CategoryID", + // }, + // }, + // }, + // }, + // }, + // }, } for _, sql := range testSqlStatements { @@ -200,6 +226,15 @@ func TestParseSelectStatement(t *testing.T) { } } + if len(answer.Joins) != len(expected.Joins) { + t.Errorf("got %d number of joins for Select.Joinss, expected %d", len(answer.Joins), len(expected.Joins)) + } else { + for i, expectedJoin := range expected.Joins { + t.Errorf("got %d for Select.Joins[%d].Type, expected %d", answer.Joins[i].Type, i, expectedJoin.Type) + } + + } + }) } }