2021-1-capstone-design2 / 2016104120

Go to a project
Toggle navigation Toggle navigation pinning
Ma Suhyeon
5007d915 1 parent 387d7f21

Implement search and graph

Expand all
Inline Side-by-side
Showing 6 changed files with 118 additions and 9 deletions
...@@ -2,9 +2,11 @@ package main ...@@ -2,9 +2,11 @@ package main
2 2
3 import ( 3 import (
4 "fmt" 4 "fmt"
5 + "net/http"
5 6
6 "github.com/labstack/echo/v4" 7 "github.com/labstack/echo/v4"
7 "github.com/labstack/echo/v4/middleware" 8 "github.com/labstack/echo/v4/middleware"
9 + "github.com/olivere/elastic/v7"
8 10
9 _ "github.com/go-sql-driver/mysql" 11 _ "github.com/go-sql-driver/mysql"
10 "github.com/jmoiron/sqlx" 12 "github.com/jmoiron/sqlx"
...@@ -19,6 +21,7 @@ const ( ...@@ -19,6 +21,7 @@ const (
19 type App struct { 21 type App struct {
20 Config Config 22 Config Config
21 db *sqlx.DB 23 db *sqlx.DB
24 + es *elastic.Client
22 echo *echo.Echo 25 echo *echo.Echo
23 } 26 }
24 27
...@@ -28,6 +31,11 @@ func NewApp(config Config) *App { ...@@ -28,6 +31,11 @@ func NewApp(config Config) *App {
28 31
29 dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s?parseTime=true", config.Database.User, config.Database.Password, config.Database.Host, config.Database.Name) 32 dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s?parseTime=true", config.Database.User, config.Database.Password, config.Database.Host, config.Database.Name)
30 app.db = sqlx.MustOpen("mysql", dsn) 33 app.db = sqlx.MustOpen("mysql", dsn)
34 + es, err := elastic.NewClient()
35 + if err != nil {
36 + panic(err)
37 + }
38 + app.es = es
31 39
32 auth := middleware.JWTWithConfig(middleware.JWTConfig{ 40 auth := middleware.JWTWithConfig(middleware.JWTConfig{
33 SigningKey: []byte(config.TokenSecret), 41 SigningKey: []byte(config.TokenSecret),
...@@ -35,6 +43,20 @@ func NewApp(config Config) *App { ...@@ -35,6 +43,20 @@ func NewApp(config Config) *App {
35 }) 43 })
36 44
37 app.echo = echo.New() 45 app.echo = echo.New()
46 +
47 + app.echo.Use(middleware.CORSWithConfig(middleware.CORSConfig{
48 + AllowMethods: []string{
49 + http.MethodGet,
50 + http.MethodPost,
51 + http.MethodPut,
52 + http.MethodDelete,
53 + http.MethodPatch,
54 + http.MethodOptions,
55 + http.MethodHead,
56 + },
57 + AllowHeaders: []string{"*"},
58 + }))
59 +
38 app.echo.POST("/users", app.PostUsers) 60 app.echo.POST("/users", app.PostUsers)
39 app.echo.POST("/users/tokens", app.PostTokens) 61 app.echo.POST("/users/tokens", app.PostTokens)
40 app.echo.POST("/extractions", app.PostExtractions, auth) 62 app.echo.POST("/extractions", app.PostExtractions, auth)
...@@ -48,6 +70,8 @@ func NewApp(config Config) *App { ...@@ -48,6 +70,8 @@ func NewApp(config Config) *App {
48 extraction.GET("/processes", app.GetProcesses) 70 extraction.GET("/processes", app.GetProcesses)
49 extraction.GET("/alarms", app.GetAlarms) 71 extraction.GET("/alarms", app.GetAlarms)
50 72
73 + app.echo.GET("/graph/:extractions", app.GetGraph)
74 +
51 return app 75 return app
52 } 76 }
53 77
......
1 package main 1 package main
2 2
3 import ( 3 import (
4 + "context"
4 "fmt" 5 "fmt"
5 "net/http" 6 "net/http"
6 "strconv" 7 "strconv"
...@@ -9,6 +10,7 @@ import ( ...@@ -9,6 +10,7 @@ import (
9 10
10 "github.com/jmoiron/sqlx" 11 "github.com/jmoiron/sqlx"
11 "github.com/labstack/echo/v4" 12 "github.com/labstack/echo/v4"
13 + "github.com/olivere/elastic/v7"
12 14
13 _ "github.com/mattn/go-sqlite3" 15 _ "github.com/mattn/go-sqlite3"
14 ) 16 )
...@@ -82,8 +84,32 @@ type Message struct { ...@@ -82,8 +84,32 @@ type Message struct {
82 func (app *App) GetMessages(c echo.Context) error { 84 func (app *App) GetMessages(c echo.Context) error {
83 messages := []Message{} 85 messages := []Message{}
84 86
85 - query := `SELECT * FROM messages WHERE extraction_no=?` 87 + no := c.Param("no")
86 - app.db.Unsafe().Select(&messages, query, c.Param("no")) 88 +
89 + q := c.QueryParam("q")
90 + if len(q) > 0 {
91 + res, err := app.es.Search("messages-" + no).
92 + Query(elastic.NewMatchQuery("content", q)).
93 + Do(context.Background())
94 +
95 + if err != nil {
96 + return err
97 + }
98 +
99 + for _, hit := range res.Hits.Hits {
100 + message := Message{}
101 +
102 + app.db.Unsafe().Get(
103 + &message,
104 + "SELECT * FROM messages WHERE extraction_no=? AND `id`=?",
105 + no, hit.Id,
106 + )
107 +
108 + messages = append(messages, message)
109 + }
110 + } else {
111 + app.db.Unsafe().Select(&messages, `SELECT * FROM messages WHERE extraction_no=?`, no)
112 + }
87 113
88 return c.JSON(http.StatusOK, messages) 114 return c.JSON(http.StatusOK, messages)
89 } 115 }
......
1 package main 1 package main
2 2
3 import ( 3 import (
4 + "context"
4 "fmt" 5 "fmt"
5 "io" 6 "io"
6 "io/ioutil" 7 "io/ioutil"
...@@ -10,6 +11,8 @@ import ( ...@@ -10,6 +11,8 @@ import (
10 "github.com/dgrijalva/jwt-go" 11 "github.com/dgrijalva/jwt-go"
11 "github.com/jmoiron/sqlx" 12 "github.com/jmoiron/sqlx"
12 "github.com/labstack/echo/v4" 13 "github.com/labstack/echo/v4"
14 +
15 + _ "github.com/mattn/go-sqlite3"
13 ) 16 )
14 17
15 func (app *App) PostExtractions(c echo.Context) error { 18 func (app *App) PostExtractions(c echo.Context) error {
...@@ -52,13 +55,10 @@ func (app *App) PostExtractions(c echo.Context) error { ...@@ -52,13 +55,10 @@ func (app *App) PostExtractions(c echo.Context) error {
52 extNo, _ := res.LastInsertId() 55 extNo, _ := res.LastInsertId()
53 56
54 rows, err := db.Queryx("SELECT * FROM calllog") 57 rows, err := db.Queryx("SELECT * FROM calllog")
55 - fmt.Println(err)
56 if err == nil { 58 if err == nil {
57 for rows.Next() { 59 for rows.Next() {
58 vals, _ := rows.SliceScan() 60 vals, _ := rows.SliceScan()
59 - fmt.Println(vals)
60 _, err = tx.Exec("INSERT INTO calls VALUES (?, ?, ?, ?, ?, ?, ?)", append([]interface{}{extNo}, vals...)...) 61 _, err = tx.Exec("INSERT INTO calls VALUES (?, ?, ?, ?, ?, ?, ?)", append([]interface{}{extNo}, vals...)...)
61 - fmt.Println(err)
62 } 62 }
63 } 63 }
64 64
...@@ -76,14 +76,30 @@ func (app *App) PostExtractions(c echo.Context) error { ...@@ -76,14 +76,30 @@ func (app *App) PostExtractions(c echo.Context) error {
76 76
77 rows, err = db.Queryx("SELECT mid, type, address, body, date FROM sms") 77 rows, err = db.Queryx("SELECT mid, type, address, body, date FROM sms")
78 if err == nil { 78 if err == nil {
79 + idxName := fmt.Sprintf("messages-%d", extNo)
80 + app.es.CreateIndex(idxName).Body(`{
81 + "settings": {
82 + "analysis": {
83 + "analyzer": {
84 + "default": {
85 + "type": "custom",
86 + "tokenizer": "nori_tokenizer"
87 + }
88 + }
89 + }
90 + }
91 + }`).Do(context.Background())
92 +
79 for rows.Next() { 93 for rows.Next() {
80 vals, _ := rows.SliceScan() 94 vals, _ := rows.SliceScan()
81 tx.Exec("INSERT INTO messages VALUES (?, ?, ?, ?, ?, ?)", append([]interface{}{extNo}, vals...)...) 95 tx.Exec("INSERT INTO messages VALUES (?, ?, ?, ?, ?, ?)", append([]interface{}{extNo}, vals...)...)
96 + app.es.Index().Index(idxName).Id(fmt.Sprint(vals[0])).BodyJson(echo.Map{"content": string(vals[3].([]byte))}).Do(context.Background())
82 } 97 }
83 } 98 }
84 99
85 rows, err = db.Queryx("SELECT PID, UID, PPID, STIME, TIME, CMD FROM process") 100 rows, err = db.Queryx("SELECT PID, UID, PPID, STIME, TIME, CMD FROM process")
86 if err == nil { 101 if err == nil {
102 +
87 for rows.Next() { 103 for rows.Next() {
88 vals, _ := rows.SliceScan() 104 vals, _ := rows.SliceScan()
89 tx.Exec("INSERT INTO processes VALUES (?, ?, ?, ?, ?, ?, ?)", append([]interface{}{extNo}, vals...)...) 105 tx.Exec("INSERT INTO processes VALUES (?, ?, ?, ?, ?, ?, ?)", append([]interface{}{extNo}, vals...)...)
......
...@@ -4,13 +4,11 @@ go 1.15 ...@@ -4,13 +4,11 @@ go 1.15
4 4
5 require ( 5 require (
6 github.com/dgrijalva/jwt-go v3.2.0+incompatible 6 github.com/dgrijalva/jwt-go v3.2.0+incompatible
7 + github.com/elastic/go-elasticsearch/v7 v7.12.0
7 github.com/go-sql-driver/mysql v1.5.0 8 github.com/go-sql-driver/mysql v1.5.0
8 - github.com/google/uuid v1.1.2
9 - github.com/gorilla/handlers v1.5.1
10 - github.com/gorilla/mux v1.8.0
11 github.com/jmoiron/sqlx v1.2.0 9 github.com/jmoiron/sqlx v1.2.0
12 - github.com/labstack/echo v3.3.10+incompatible
13 github.com/labstack/echo/v4 v4.2.2 10 github.com/labstack/echo/v4 v4.2.2
14 github.com/mattn/go-sqlite3 v1.9.0 11 github.com/mattn/go-sqlite3 v1.9.0
12 + github.com/olivere/elastic/v7 v7.0.24
15 golang.org/x/crypto v0.0.0-20201217014255-9d1352758620 13 golang.org/x/crypto v0.0.0-20201217014255-9d1352758620
16 ) 14 )
......
This diff is collapsed. Click to expand it.
1 +package main
2 +
3 +import (
4 + "net/http"
5 + "strconv"
6 + "strings"
7 +
8 + "github.com/labstack/echo/v4"
9 +)
10 +
11 +func (app *App) GetGraph(c echo.Context) error {
12 + sep := strings.Split(c.Param("extractions"), ",")
13 + extractions := map[string]uint64{}
14 + for _, s := range sep {
15 + var phone string
16 + app.db.Get(&phone, "SELECT phone FROM extractions WHERE `no`=?", s)
17 + extractions[phone], _ = strconv.ParseUint(s, 10, 64)
18 + }
19 +
20 + calls := map[[2]string]int{}
21 + for p, e := range extractions {
22 + rows, _ := app.db.Query("SELECT `number`, COUNT(1) FROM calls WHERE extraction_no=? GROUP BY `number`", e)
23 + for rows.Next() {
24 + var number string
25 + var count int
26 + rows.Scan(&number, &count)
27 +
28 + if _, ok := calls[[2]string{p, number}]; ok {
29 + continue
30 + }
31 +
32 + calls[[2]string{p, number}] = count
33 + calls[[2]string{number, p}] = count
34 + }
35 + }
36 +
37 + result := []interface{}{}
38 + for k, c := range calls {
39 + if k[0] < k[1] && c >= 5 {
40 + result = append(result, echo.Map{"numbers": k, "calls": c})
41 + }
42 + }
43 +
44 + return c.JSON(http.StatusOK, result)
45 +}