Beitzah/query-interpreter
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: created simple api for client testing

Browse Source
This commit is contained in:
Yehoshua Sandler 2025-04-27 14:29:20 -05:00
parent 0522b90917
commit 60b1033dba
7 changed files with 410 additions and 159 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.env.example Normal file
View File

@ -0,0 +1 @@
PORT=6796

121
README.md
View File

@ -6,6 +6,68 @@
Core program to interpret query language strings into structured data, and back again.
## How to Use the Project
### Microservice for client applications
At least for now, this can be treated like a micro service. Very simply you query the endpoint(s)
with your SQL strings and retrieve the structured data back.
```
POST /query
body: {
sql: string
}
```
Be aware that the api is not currently converting enums to their string representations, so you might expect
JoinType to be `"INNER"`, bit it is returned as `0`. Refer to the [dto](q/dto.go) as a reference to what the
`string` values for these `enum`s would be.
> `iota` is incrementing for all values in the `enum`, and starts at `0` unless modified. `iota + 1` will start at `1`
Right now we are only parsing SELECT statements. If you try to do something else it will either error out
or hang. The HTTP response should timeout after 30 seconds.
### Development on core logic
This project is wored on via TDD methods, it is the only way to do so as the parsing of SQL is so janky. If
you are wanting to add a feature to the parsing, you need to first write a unit test.
Become familiar with [select_test](q/select_test.md) to see how we are doing it. In Brief:
We have the test struct where `input` is the entire SQL string you are testing, and expected is
the exact struct (of which ever query struct) you expect to see returned.
```go
type ParsingTest struct {
input string
expected Select
}
```
Add yours to the `var testSqlStatements []ParsingTest` of the file.
If you are adding a new field to the query's struct, or modifying any fields, make sure to add or update the
conditionals in teh `t.Run(testName, func(t *testing.T)` block.
#### Remember the TDD Process
- Write enough of a test to make sure it fails
- Write enough prod code to make sure it passes
- Repeat until finished developing the feature
### Starting the app
**Prerequisites:**
* Go installed (version X.Y or higher - check your code for specifics)
* `go mod tidy` to fetch dependencies
* `cp .env.example .env` to create your own .env file
**Running the App:**
1. `go run main.go` to start the server (PORT is determined by the .env file)
2. `go test ./q` to run test suite if developing features (add `-v` if you want a verbose output)
## Data Structure Philosophy
We are operating off of the philosophy that the first class data is SQL Statement stings.
@ -68,19 +130,43 @@ already been processed through the scan. Making naming and reading new flags tri
A `Select` object is shaped as the following:
```go
type Select struct {
Table string
Columns []Column
Conditionals []Conditional
OrderBys []OrderBy
Joins []Join
IsWildcard bool
IsDistinct bool
Table string `json:"table"`
Columns []Column `json:"columns"`
Conditionals []Conditional `json:"conditionals"`
OrderBys []OrderBy `json:"order_bys"`
Joins []Join `json:"joins"`
IsWildcard bool `json:"is_wildcard"`
IsDistinct bool `json:"is_distinct"`
}
type Column struct {
Name string
Alias string
AggregateFunction AggregateFunctionType
Name string `json:"name"`
Alias string `json:"alias"`
AggregateFunction AggregateFunctionType `json:"aggregate_function"` // Changed type name to match Go naming conventions
}
type Conditional struct {
Key string `json:"key"`
Operator string `json:"operator"`
Value string `json:"value"`
Extension string `json:"extension"`
}
type OrderBy struct {
Key string
IsDescend bool // SQL queries with no ASC|DESC on their ORDER BY are ASC by default, hence why this bool for the opposite
}
type Join struct {
Type JoinType `json:"type"`
Table Table `json:"table"`
Ons []Conditional `json:"ons"`
}
// Only used in Join.Table right now, but Select.Table will also use this soon
type Table struct {
Name string `json:"name"`
Alias string `json:"alias"`
}
type AggregateFunctionType int
@ -92,23 +178,8 @@ const (
AVG
)
//dependency in query.go
type Conditional struct {
Key string
Operator string
Value string
DataType string
Extension string // AND, OR, etc
}
type OrderBy struct {
Key string
IsDescend bool // SQL queries with no ASC|DESC on their ORDER BY are ASC by default, hence why this bool for the opposite
}
```
## Improvement Possibilities
- Maybe utilize the `lookBehindBuffer` more to cut down the number of state flags in the scans?

102
main.go
View File

@ -1,28 +1,41 @@
package main
import (
"context"
"encoding/json"
"fmt"
//"query-inter/q"
"io"
"os"
"github.com/DataDog/go-sqllexer"
// "github.com/DataDog/go-sqllexer"
//"github.com/DataDog/go-sqllexer"
"log"
"net/http"
"query-inter/q"
"time"
)
func main() {
selectQuery := "SELECT MIN(Price) AS SmallestPrice, CategoryID FROM Products GROUP BY CategoryID;"
//selectQuery := "SELECT MIN(Price) AS SmallestPrice, CategoryID FROM Products GROUP BY CategoryID;"
PORT := os.Getenv("PORT")
if PORT == "" {
log.Panicln("PORT could not be read from env")
}
StartServer(PORT)
//allStatements := q.ExtractSqlStatmentsFromString(selectQuery)
//fmt.Println(allStatements)
lexer := sqllexer.New(selectQuery)
for {
token := lexer.Scan()
fmt.Println(token.Value, token.Type)
if token.Type == sqllexer.EOF {
break
}
}
//lexer := sqllexer.New(selectQuery)
//for {
// token := lexer.Scan()
// fmt.Println(token.Value, token.Type)
//
// if token.Type == sqllexer.EOF {
// break
// }
//}
//for _, sql := range allStatements {
//query := q.ParseSelectStatement(sql)
@ -32,3 +45,66 @@ func main() {
//}
}
func StartServer(port string) {
http.HandleFunc("/query", HandlePostQuery)
fmt.Println("Starting Server on 8080")
fmt.Println("call POST /query with { sql: string }")
log.Fatal(http.ListenAndServe(":"+port, nil))
}
type QueryRequest struct {
SQL string `json:"sql"`
}
func HandlePostQuery(w http.ResponseWriter, r *http.Request) {
body, err := io.ReadAll(r.Body)
if err != nil {
http.Error(w, "Error reading request body", http.StatusBadRequest)
return
}
defer r.Body.Close()
var request QueryRequest
err = json.Unmarshal(body, &request)
if err != nil {
http.Error(w, "Error parsing JSON: "+string(body), http.StatusBadRequest)
return
}
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
defer cancel()
var query q.Select
done := make(chan struct{})
go func() {
defer cancel()
query = q.ParseSelectStatement(request.SQL)
done <- struct{}{}
}()
select {
case <-done:
// Proceed
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK) // Set the HTTP status code
jsonData, err := q.MarshalSelect(query)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
_, err = w.Write(jsonData)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
case <-ctx.Done():
http.Error(w, "Request timed out", http.StatusRequestTimeout)
return
}
}

187
q/dto.go Normal file
View File

@ -0,0 +1,187 @@
package q
import (
"encoding/json"
"strings"
)
const NONE = 0
type QueryType int
const (
SELECT QueryType = iota + 1
UPDATE
INSERT
DELETE
)
// QueryTypeString converts a QueryType integer to its string representation.
func QueryTypeString(t QueryType) string {
switch t {
case SELECT:
return "SELECT"
case UPDATE:
return "UPDATE"
case INSERT:
return "INSERT"
case DELETE:
return "DELETE"
default:
return "UNKNOWN"
}
}
type Table struct {
Name string `json:"name"`
Alias string `json:"alias"`
}
type Conditional struct {
Key string `json:"key"`
Operator string `json:"operator"`
Value string `json:"value"`
Extension string `json:"extension"`
}
type AggregateFunctionType int
const (
MIN AggregateFunctionType = iota + 1
MAX
COUNT
SUM
AVG
)
// AggregateFunctionTypeString converts an AggregateFunctionType integer to its string representation.
func AggregateFunctionTypeString(t AggregateFunctionType) string {
switch t {
case MIN:
return "MIN"
case MAX:
return "MAX"
case COUNT:
return "COUNT"
case SUM:
return "SUM"
case AVG:
return "AVG"
default:
return "UNKNOWN"
}
}
// Converts the string name of AggregateFunctionType into it original int
func AggregateFunctionTypeByName(name string) AggregateFunctionType {
var functionType AggregateFunctionType
switch strings.ToUpper(name) {
case "MIN":
functionType = MIN
case "MAX":
functionType = MAX
case "COUNT":
functionType = COUNT
case "SUM":
functionType = SUM
case "AVG":
functionType = AVG
default:
functionType = 0
}
return functionType
}
type JoinType int
const (
INNER JoinType = iota
LEFT
RIGHT
FULL
SELF
)
// JoinTypeString converts a JoinType integer to its string representation.
func JoinTypeString(t JoinType) string {
switch t {
case INNER:
return "INNER"
case LEFT:
return "LEFT"
case RIGHT:
return "RIGHT"
case FULL:
return "FULL"
case SELF:
return "SELF"
default:
return "UNKNOWN"
}
}
type Column struct {
Name string `json:"name"`
Alias string `json:"alias"`
AggregateFunction AggregateFunctionType `json:"aggregate_function"` // Changed type name to match Go naming conventions
}
type Join struct {
Type JoinType `json:"type"`
Table Table `json:"table"`
Ons []Conditional `json:"ons"`
}
type Select struct {
Table string `json:"table"`
Columns []Column `json:"columns"`
Conditionals []Conditional `json:"conditionals"`
OrderBys []OrderBy `json:"order_bys"`
Joins []Join `json:"joins"`
IsWildcard bool `json:"is_wildcard"`
IsDistinct bool `json:"is_distinct"`
}
type OrderBy struct {
Key string `json:"key"`
IsDescend bool `json:"is_descend"`
}
func MarshalSelect(selectStatement Select) ([]byte, error) {
jsonData, err := json.MarshalIndent(selectStatement, "", " ")
return jsonData, err
}
//func test() {
//selectStatement := Select{
// Table: "users",
// Columns: []Column{
// {Name: "id", Alias: "user_id", AggregateFunction: COUNT},
// },
// Conditionals: []Conditional{
// {Key: "age", Operator: ">", Value: "18"},
// },
// OrderBys: []OrderBy{
// {Key: "name", IsDescend: false},
// },
// Joins: []Join{
// {
// Type: LEFT,
// Table: Table{Name: "orders"},
// Ons: []Conditional{{Key: "user_id", Operator: "=", Value: "users.id"}},
// }},
//}
//
//jsonData, err := json.MarshalIndent(selectStatement, "", " ")
//if err != nil {
// fmt.Println("Error marshaling JSON:", err)
// return
//}
//
//fmt.Println(string(jsonData))
//
//fmt.Println(QueryTypeString(SELECT))
//fmt.Println(AggregateFunctionTypeString(MAX))
//fmt.Println(JoinTypeString(INNER))
//}

18
q/query.go
View File

@ -10,24 +10,6 @@ type Query interface {
GetFullSql() string
}
const NONE = 0
type QueryType int
const (
SELECT QueryType = iota + 1
UPDATE
INSERT
DELETE
)
type Conditional struct {
Key string
Operator string
Value string
Extension string // AND, OR, etc
}
func GetQueryTypeFromToken(token *sqllexer.Token) QueryType {
if token.Type != sqllexer.COMMAND {
return NONE

105
q/select.go
View File

@ -7,106 +7,13 @@ import (
"github.com/DataDog/go-sqllexer"
)
type AggregateFunctionType int
const (
MIN AggregateFunctionType = iota + 1
MAX
COUNT
SUM
AVG
)
type JoinType int
const (
INNER JoinType = iota
LEFT
RIGHT
FULL
SELF
)
type Table struct {
Name string
Alias string
}
type Column struct {
Name string
Alias string
AggregateFunction AggregateFunctionType
}
type Join struct {
Type JoinType
Table Table
Ons []Conditional
}
type Select struct {
Table string
Columns []Column
Conditionals []Conditional
OrderBys []OrderBy
Joins []Join
IsWildcard bool
IsDistinct bool
}
type OrderBy struct {
Key string
IsDescend bool // SQL queries with no ASC|DESC on their ORDER BY are ASC by default, hence why this bool for the opposite
}
func GetAggregateFunctionTypeByName(name string) AggregateFunctionType {
var functionType AggregateFunctionType
switch strings.ToUpper(name) {
case "MIN":
functionType = MIN
case "MAX":
functionType = MAX
case "COUNT":
functionType = COUNT
case "SUM":
functionType = SUM
case "AVG":
functionType = AVG
default:
functionType = 0
}
return functionType
}
func GetAggregateFunctionNameByType(functionType AggregateFunctionType) string {
var functionName string
switch functionType {
case MIN:
functionName = "MIN"
case MAX:
functionName = "MAX"
case COUNT:
functionName = "COUNT"
case SUM:
functionName = "SUM"
case AVG:
functionName = "AVG"
default:
functionName = ""
}
return functionName
}
func GetFullStringFromColumn(column Column) string {
var workingSlice string
if column.AggregateFunction > 0 {
workingSlice = fmt.Sprintf(
"%s(%s)",
GetAggregateFunctionNameByType(column.AggregateFunction),
AggregateFunctionTypeString(column.AggregateFunction),
column.Name,
)
} else {
@ -140,7 +47,6 @@ func (q *Select) GetFullSql() string {
workingSqlSlice = append(workingSqlSlice, "FROM "+q.Table)
// TODO: need to account for `AND` and `OR`s and stuff
for _, condition := range q.Conditionals {
workingSqlSlice = append(workingSqlSlice, condition.Key)
workingSqlSlice = append(workingSqlSlice, condition.Operator)
@ -152,13 +58,6 @@ func (q *Select) GetFullSql() string {
return fullSql
}
func mutateSelectFromKeyword(query *Select, keyword string) {
switch strings.ToUpper(keyword) {
case "DISTINCT":
query.IsDistinct = true
}
}
func unshiftBuffer(buf *[10]sqllexer.Token, value sqllexer.Token) {
for i := 9; i >= 1; i-- {
buf[i] = buf[i-1]
@ -215,7 +114,7 @@ func ParseSelectStatement(sql string) Select {
continue
} else if token.Type == sqllexer.FUNCTION {
unshiftBuffer(&lookBehindBuffer, *token)
workingColumn.AggregateFunction = GetAggregateFunctionTypeByName(token.Value)
workingColumn.AggregateFunction = AggregateFunctionTypeByName(token.Value)
continue
} else if token.Type == sqllexer.PUNCTUATION {
if token.Value == "," {

35
q/select_test.go
View File

@ -137,6 +137,32 @@ func TestParseSelectStatement(t *testing.T) {
},
},
},
// {
// input: "SELECT ProductID, ProductName, CategoryName FROM Products INNER JOIN Categories ON Products.CategoryID = Categories.CategoryID; ",
// expected: Select{
// Table: "Products",
// Columns: []Column{
// {Name: "ProductID"},
// {Name: "ProductName"},
// {Name: "CategoryName"},
// },
// Joins: []Join{
// {
// Type: INNER,
// Table: Table{
// Name: "Categories",
// },
// Ons: []Conditional{
// {
// Key: "Products.CategoryID",
// Operator: "=",
// Value: "Categories.CategoryID",
// },
// },
// },
// },
// },
// },
}
for _, sql := range testSqlStatements {
@ -200,6 +226,15 @@ func TestParseSelectStatement(t *testing.T) {
}
}
if len(answer.Joins) != len(expected.Joins) {
t.Errorf("got %d number of joins for Select.Joinss, expected %d", len(answer.Joins), len(expected.Joins))
} else {
for i, expectedJoin := range expected.Joins {
t.Errorf("got %d for Select.Joins[%d].Type, expected %d", answer.Joins[i].Type, i, expectedJoin.Type)
}
}
})
}
}