2021-1-capstone-design2 / 2016104120

Go to a project
Toggle navigation Toggle navigation pinning
Ma Suhyeon
b8d2cfcd 1 parent 5c02095d

Implement phone normalization 

Implement number filter
Implement daily stats
Implement put APIs
Inline Side-by-side
Showing 4 changed files with 171 additions and 28 deletions
......@@ -66,22 +66,26 @@ func NewApp(config Config) *App {
extraction.GET("/calls", app.GetCalls)
extraction.POST("/calls", app.PostCall)
extraction.PUT("/calls/:id", app.PutCall)
extraction.DELETE("/calls/:id", app.DeleteCall)
extraction.GET("/messages", app.GetMessages)
extraction.POST("/messages", app.PostMessage)
extraction.PUT("/messages/:id", app.PutMessage)
extraction.DELETE("/messages/:id", app.DeleteMessage)
extraction.GET("/calls/analyses", app.GetCallsAnalyses)
extraction.GET("/apps/analyses", app.GetAppsAnalyses)
extraction.POST("/apps/analyses", app.PostAppAnalysis)
extraction.PUT("/apps/analyses/:package", app.PutAppAnalysis)
extraction.DELETE("/apps/analyses/:package", app.DeleteAppAnalysis)
extraction.GET("/messages/analyses", app.GetMessagesAnalyses)
extraction.GET("/processes", app.GetProcesses)
extraction.GET("/alarms", app.GetAlarms)
extraction.GET("/schedules", app.GetSchedules)
extraction.GET("/dailies", app.GetDailyContacts)
app.echo.GET("/graph/:extractions", app.GetGraph)
......
......@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"net/http"
"strconv"
"time"
"github.com/labstack/echo/v4"
......@@ -16,7 +17,7 @@ type Call struct {
ID int `json:"id" db:"id"`
Type int `json:"type" db:"type"`
Name *string `json:"name" db:"name"`
Number int `json:"number" db:"number"`
Number string `json:"number" db:"number"`
Duration int `json:"duration" db:"duration"`
Date time.Time `json:"date" db:"date"`
}
......@@ -25,7 +26,15 @@ func (app *App) GetCalls(c echo.Context) error {
calls := []Call{}
query := `SELECT * FROM calls WHERE extraction_no=?`
app.db.Unsafe().Select(&calls, query, c.Param("no"))
vals := []interface{}{c.Param("no")}
number := c.QueryParam("number")
if len(number) > 0 {
query += " AND `number`=?"
vals = append(vals, NormalizeNumber(number))
}
app.db.Unsafe().Select(&calls, query, vals...)
return c.JSON(http.StatusOK, calls)
}
......@@ -45,6 +54,28 @@ func (app *App) PostCall(c echo.Context) error {
return c.NoContent(http.StatusNoContent)
}
func (app *App) PutCall(c echo.Context) error {
call := Call{}
if err := c.Bind(&call); err != nil {
return err
}
query := "UPDATE calls SET `id`=?, `type`=?, `name`=?, `number`=?, `duration`=?, `date`=? " +
"WHERE `extraction_no`=? AND `id`=?"
res, err := app.db.Exec(query, call.ID, call.Type, call.Name, call.Number, call.Duration, call.Date, c.Param("no"), c.Param("id"))
if err != nil {
return nil
}
if rows, err := res.RowsAffected(); err != nil {
return err
} else if rows == 0 {
return echo.NewHTTPError(http.StatusNotFound, "Can not find the call")
} else {
return c.NoContent(http.StatusNoContent)
}
}
func (app *App) DeleteCall(c echo.Context) error {
query := "DELETE FROM calls WHERE `extraction_no`=? AND `id`=?"
res, err := app.db.Exec(query, c.Param("no"), c.Param("id"))
......@@ -120,6 +151,34 @@ func (app *App) PostAppAnalysis(c echo.Context) error {
return c.NoContent(http.StatusNoContent)
}
func (app *App) PutAppAnalysis(c echo.Context) error {
info := AppInfo{}
if err := c.Bind(&info); err != nil {
return err
}
query := "UPDATE apps SET `package_name`=?, `name`=?, `version`=?, " +
"`wifi_usage`=?, `cellular_usage`=?, `last_used`=?, `foreground_time`=? " +
"WHERE `extraction_no`=? AND `package_name`=?"
res, err := app.db.Exec(
query,
info.PackageName, info.Name, info.Version,
info.WifiUsage, info.CellularUsage, info.LastUsed, info.ForegroundTime,
c.Param("no"), c.Param("package"),
)
if err != nil {
return err
}
if rows, err := res.RowsAffected(); err != nil {
return err
} else if rows == 0 {
return echo.NewHTTPError(http.StatusNotFound, "Can not find the app")
} else {
return c.NoContent(http.StatusNoContent)
}
}
func (app *App) DeleteAppAnalysis(c echo.Context) error {
query := "DELETE FROM apps WHERE `extraction_no`=? AND `package_name`=?"
res, err := app.db.Exec(query, c.Param("no"), c.Param("package"))
......@@ -149,6 +208,15 @@ func (app *App) GetMessages(c echo.Context) error {
no := c.Param("no")
query := "SELECT * FROM messages WHERE extraction_no=?"
vals := []interface{}{no}
number := c.QueryParam("number")
if len(number) > 0 {
query += " AND `address`=?"
vals = append(vals, NormalizeNumber(number))
}
q := c.QueryParam("q")
if len(q) > 0 {
res, err := app.es.Search("messages-" + no).
......@@ -159,19 +227,22 @@ func (app *App) GetMessages(c echo.Context) error {
return err
}
query += " AND `id`=?"
for _, hit := range res.Hits.Hits {
message := Message{}
app.db.Unsafe().Get(
err := app.db.Unsafe().Get(
&message,
"SELECT * FROM messages WHERE extraction_no=? AND `id`=?",
no, hit.Id,
query, append(vals, hit.Id)...,
)
if err == nil {
messages = append(messages, message)
}
}
} else {
app.db.Unsafe().Select(&messages, `SELECT * FROM messages WHERE extraction_no=?`, no)
app.db.Unsafe().Select(&messages, query, vals...)
}
return c.JSON(http.StatusOK, messages)
......@@ -193,6 +264,29 @@ func (app *App) PostMessage(c echo.Context) error {
return c.NoContent(http.StatusNoContent)
}
func (app *App) PutMessage(c echo.Context) error {
message := Message{}
if err := c.Bind(&message); err != nil {
return err
}
query := "UPDATE messages SET `id`=?, `type`=?, `address`=?, `body`=?, `date`=? " +
"WHERE `extraction_no`=? AND `id`=?"
res, err := app.db.Exec(query, message.ID, message.Type, message.Address, message.Body, message.Date, c.Param("no"), c.Param("id"))
if err != nil {
return err
}
if rows, err := res.RowsAffected(); err != nil {
return err
} else if rows == 0 {
return echo.NewHTTPError(http.StatusNotFound, "Can not find the message")
} else {
return c.NoContent(http.StatusNoContent)
}
}
func (app *App) DeleteMessage(c echo.Context) error {
query := "DELETE FROM messages WHERE `extraction_no`=? AND `id`=?"
res, err := app.db.Exec(query, c.Param("no"), c.Param("id"))
......@@ -291,3 +385,33 @@ func (app *App) GetSchedules(c echo.Context) error {
return c.JSON(http.StatusOK, schedules)
}
type DailyContact struct {
Date time.Time `json:"date" db:"date"`
Calls int `json:"calls" db:"calls"`
Messages int `json:"messages" db:"messages"`
}
func (app *App) GetDailyContacts(c echo.Context) error {
query := "SELECT `date`, `calls`, `messages` FROM daily_contacts WHERE `extraction_no`=?"
result := []DailyContact{}
if err := app.db.Select(&result, query, c.Param("no")); err != nil {
fmt.Println(err)
return err
}
return c.JSON(http.StatusOK, result)
}
func NormalizeNumber(number string) string {
if len(number) > 0 && number[0] == '+' {
return number
}
integer, err := strconv.Atoi(number)
if err != nil {
return number
}
return fmt.Sprintf("+82%d", integer)
}
......
......@@ -83,7 +83,8 @@ func (app *App) PostExtractions(c echo.Context) error {
if err == nil {
for rows.Next() {
vals, _ := rows.SliceScan()
_, err = tx.Exec("INSERT INTO calls VALUES (?, ?, ?, ?, ?, ?, ?)", append([]interface{}{extNo}, vals...)...)
vals[3] = NormalizeNumber(string(vals[3].([]byte)))
tx.Exec("INSERT INTO calls VALUES (?, ?, ?, ?, ?, ?, ?)", append([]interface{}{extNo}, vals...)...)
}
}
......@@ -117,6 +118,7 @@ func (app *App) PostExtractions(c echo.Context) error {
for rows.Next() {
vals, _ := rows.SliceScan()
vals[2] = NormalizeNumber(string(vals[2].([]byte)))
tx.Exec("INSERT INTO messages VALUES (?, ?, ?, ?, ?, ?)", append([]interface{}{extNo}, vals...)...)
app.es.Index().Index(idxName).Id(fmt.Sprint(vals[0])).BodyJson(echo.Map{"content": string(vals[3].([]byte))}).Do(context.Background())
}
......
......@@ -27,36 +27,49 @@ cursor = db.cursor()
ext_no = int(sys.argv[1])
cursor.execute("SELECT `type`, `number`, `duration` FROM calls WHERE `extraction_no`=%s", (ext_no))
cursor.execute("SELECT `type`, `number`, `duration`, `date` FROM calls WHERE `extraction_no`=%s", (ext_no))
calls = cursor.fetchall()
cursor.execute("SELECT `type`, `address` FROM messages WHERE `extraction_no`=%s", (ext_no))
cursor.execute("SELECT `type`, `address`, `date` FROM messages WHERE `extraction_no`=%s", (ext_no))
messages = cursor.fetchall()
regions = {
'02': 'Seoul',
'031': 'Gyeonggi',
'032': 'Incheon',
'033': 'Gangwon',
'041': 'Chungnam',
'042': 'Daejeon',
'043': 'Chungbuk',
'044': 'Sejong',
'051': 'Busan',
'052': 'Ulsan',
'053': 'Daegu',
'054': 'Gyeongbuk',
'055': 'Gyeongnam',
'061': 'Jeonnam',
'062': 'Gwangju',
'063': 'Jeonbuk',
'064': 'Jeju'
'+822': 'Seoul',
'+8231': 'Gyeonggi',
'+8232': 'Incheon',
'+8233': 'Gangwon',
'+8241': 'Chungnam',
'+8242': 'Daejeon',
'+8243': 'Chungbuk',
'+8244': 'Sejong',
'+8251': 'Busan',
'+8252': 'Ulsan',
'+8253': 'Daegu',
'+8254': 'Gyeongbuk',
'+8255': 'Gyeongnam',
'+8261': 'Jeonnam',
'+8262': 'Gwangju',
'+8263': 'Jeonbuk',
'+8264': 'Jeju'
}
spark = SparkSession.builder.getOrCreate()
cdf = spark.createDataFrame(list(calls), schema=['type', 'number', 'duration'])
mdf = spark.createDataFrame(list(messages), schema=['type', 'address'])
cdf = spark.createDataFrame(list(calls), schema=['type', 'number', 'duration', 'date'])
mdf = spark.createDataFrame(list(messages), schema=['type', 'address', 'date'])
dfdc = cdf.select(F.to_date(F.col('date')).alias('date')).groupBy('date').agg(F.count('*').alias('calls'))
dfdm = mdf.select(F.to_date(F.col('date')).alias('date')).groupBy('date').agg(F.count('*').alias('messages'))
dfd = dfdc.join(dfdm, ['date'], 'fullouter').select(
'date',
F.coalesce('calls', F.lit(0)).alias('calls'),
F.coalesce('messages', F.lit(0)).alias('messages')
)
for r in dfd.collect():
sql = "INSERT INTO daily_contacts VALUES (%s, %s, %s, %s)"
cursor.execute(sql, (ext_no, r[0], r[1], r[2]))
result = None
for key, val in regions.items():
......