Implement phone normalization

Implement number filter
Implement daily stats
Implement put APIs

@@ -66,22 +66,26 @@ func NewApp(config Config) *App {
 
 
 	extraction.GET("/calls", app.GetCalls)
 	extraction.GET("/calls", app.GetCalls)
 	extraction.POST("/calls", app.PostCall)
 	extraction.POST("/calls", app.PostCall)
+	extraction.PUT("/calls/:id", app.PutCall)
 	extraction.DELETE("/calls/:id", app.DeleteCall)
 	extraction.DELETE("/calls/:id", app.DeleteCall)
 
 
 	extraction.GET("/messages", app.GetMessages)
 	extraction.GET("/messages", app.GetMessages)
 	extraction.POST("/messages", app.PostMessage)
 	extraction.POST("/messages", app.PostMessage)
+	extraction.PUT("/messages/:id", app.PutMessage)
 	extraction.DELETE("/messages/:id", app.DeleteMessage)
 	extraction.DELETE("/messages/:id", app.DeleteMessage)
 
 
 	extraction.GET("/calls/analyses", app.GetCallsAnalyses)
 	extraction.GET("/calls/analyses", app.GetCallsAnalyses)
 
 
 	extraction.GET("/apps/analyses", app.GetAppsAnalyses)
 	extraction.GET("/apps/analyses", app.GetAppsAnalyses)
 	extraction.POST("/apps/analyses", app.PostAppAnalysis)
 	extraction.POST("/apps/analyses", app.PostAppAnalysis)
+	extraction.PUT("/apps/analyses/:package", app.PutAppAnalysis)
 	extraction.DELETE("/apps/analyses/:package", app.DeleteAppAnalysis)
 	extraction.DELETE("/apps/analyses/:package", app.DeleteAppAnalysis)
 
 
 	extraction.GET("/messages/analyses", app.GetMessagesAnalyses)
 	extraction.GET("/messages/analyses", app.GetMessagesAnalyses)
 	extraction.GET("/processes", app.GetProcesses)
 	extraction.GET("/processes", app.GetProcesses)
 	extraction.GET("/alarms", app.GetAlarms)
 	extraction.GET("/alarms", app.GetAlarms)
 	extraction.GET("/schedules", app.GetSchedules)
 	extraction.GET("/schedules", app.GetSchedules)
+	extraction.GET("/dailies", app.GetDailyContacts)
 
 
 	app.echo.GET("/graph/:extractions", app.GetGraph)
 	app.echo.GET("/graph/:extractions", app.GetGraph)
 
 

@@ -4,6 +4,7 @@ import (
 	"context"
 	"context"
 	"fmt"
 	"fmt"
 	"net/http"
 	"net/http"
+	"strconv"
 	"time"
 	"time"
 
 
 	"github.com/labstack/echo/v4"
 	"github.com/labstack/echo/v4"
@@ -16,7 +17,7 @@ type Call struct {
 	ID       int       `json:"id" db:"id"`
 	ID       int       `json:"id" db:"id"`
 	Type     int       `json:"type" db:"type"`
 	Type     int       `json:"type" db:"type"`
 	Name     *string   `json:"name" db:"name"`
 	Name     *string   `json:"name" db:"name"`
-	Number   int       `json:"number" db:"number"`
+	Number   string    `json:"number" db:"number"`
 	Duration int       `json:"duration" db:"duration"`
 	Duration int       `json:"duration" db:"duration"`
 	Date     time.Time `json:"date" db:"date"`
 	Date     time.Time `json:"date" db:"date"`
 }
 }
@@ -25,7 +26,15 @@ func (app *App) GetCalls(c echo.Context) error {
 	calls := []Call{}
 	calls := []Call{}
 
 
 	query := `SELECT * FROM calls WHERE extraction_no=?`
 	query := `SELECT * FROM calls WHERE extraction_no=?`
-	app.db.Unsafe().Select(&calls, query, c.Param("no"))
+	vals := []interface{}{c.Param("no")}
+
+	number := c.QueryParam("number")
+	if len(number) > 0 {
+		query += " AND `number`=?"
+		vals = append(vals, NormalizeNumber(number))
+	}
+
+	app.db.Unsafe().Select(&calls, query, vals...)
 
 
 	return c.JSON(http.StatusOK, calls)
 	return c.JSON(http.StatusOK, calls)
 }
 }
@@ -45,6 +54,28 @@ func (app *App) PostCall(c echo.Context) error {
 	return c.NoContent(http.StatusNoContent)
 	return c.NoContent(http.StatusNoContent)
 }
 }
 
 
+func (app *App) PutCall(c echo.Context) error {
+	call := Call{}
+	if err := c.Bind(&call); err != nil {
+		return err
+	}
+
+	query := "UPDATE calls SET `id`=?, `type`=?, `name`=?, `number`=?, `duration`=?, `date`=? " +
+		"WHERE `extraction_no`=? AND `id`=?"
+	res, err := app.db.Exec(query, call.ID, call.Type, call.Name, call.Number, call.Duration, call.Date, c.Param("no"), c.Param("id"))
+	if err != nil {
+		return nil
+	}
+
+	if rows, err := res.RowsAffected(); err != nil {
+		return err
+	} else if rows == 0 {
+		return echo.NewHTTPError(http.StatusNotFound, "Can not find the call")
+	} else {
+		return c.NoContent(http.StatusNoContent)
+	}
+
+}
 func (app *App) DeleteCall(c echo.Context) error {
 func (app *App) DeleteCall(c echo.Context) error {
 	query := "DELETE FROM calls WHERE `extraction_no`=? AND `id`=?"
 	query := "DELETE FROM calls WHERE `extraction_no`=? AND `id`=?"
 	res, err := app.db.Exec(query, c.Param("no"), c.Param("id"))
 	res, err := app.db.Exec(query, c.Param("no"), c.Param("id"))
@@ -120,6 +151,34 @@ func (app *App) PostAppAnalysis(c echo.Context) error {
 	return c.NoContent(http.StatusNoContent)
 	return c.NoContent(http.StatusNoContent)
 }
 }
 
 
+func (app *App) PutAppAnalysis(c echo.Context) error {
+	info := AppInfo{}
+	if err := c.Bind(&info); err != nil {
+		return err
+	}
+
+	query := "UPDATE apps SET `package_name`=?, `name`=?, `version`=?, " +
+		"`wifi_usage`=?, `cellular_usage`=?, `last_used`=?, `foreground_time`=? " +
+		"WHERE `extraction_no`=? AND `package_name`=?"
+	res, err := app.db.Exec(
+		query,
+		info.PackageName, info.Name, info.Version,
+		info.WifiUsage, info.CellularUsage, info.LastUsed, info.ForegroundTime,
+		c.Param("no"), c.Param("package"),
+	)
+	if err != nil {
+		return err
+	}
+
+	if rows, err := res.RowsAffected(); err != nil {
+		return err
+	} else if rows == 0 {
+		return echo.NewHTTPError(http.StatusNotFound, "Can not find the app")
+	} else {
+		return c.NoContent(http.StatusNoContent)
+	}
+}
+
 func (app *App) DeleteAppAnalysis(c echo.Context) error {
 func (app *App) DeleteAppAnalysis(c echo.Context) error {
 	query := "DELETE FROM apps WHERE `extraction_no`=? AND `package_name`=?"
 	query := "DELETE FROM apps WHERE `extraction_no`=? AND `package_name`=?"
 	res, err := app.db.Exec(query, c.Param("no"), c.Param("package"))
 	res, err := app.db.Exec(query, c.Param("no"), c.Param("package"))
@@ -149,6 +208,15 @@ func (app *App) GetMessages(c echo.Context) error {
 
 
 	no := c.Param("no")
 	no := c.Param("no")
 
 
+	query := "SELECT * FROM messages WHERE extraction_no=?"
+	vals := []interface{}{no}
+
+	number := c.QueryParam("number")
+	if len(number) > 0 {
+		query += " AND `address`=?"
+		vals = append(vals, NormalizeNumber(number))
+	}
+
 	q := c.QueryParam("q")
 	q := c.QueryParam("q")
 	if len(q) > 0 {
 	if len(q) > 0 {
 		res, err := app.es.Search("messages-" + no).
 		res, err := app.es.Search("messages-" + no).
@@ -159,19 +227,22 @@ func (app *App) GetMessages(c echo.Context) error {
 			return err
 			return err
 		}
 		}
 
 
+		query += " AND `id`=?"
+
 		for _, hit := range res.Hits.Hits {
 		for _, hit := range res.Hits.Hits {
 			message := Message{}
 			message := Message{}
 
 
-			app.db.Unsafe().Get(
+			err := app.db.Unsafe().Get(
 				&message,
 				&message,
-				"SELECT * FROM messages WHERE extraction_no=? AND `id`=?",
-				no, hit.Id,
+				query, append(vals, hit.Id)...,
 			)
 			)
 
 
-			messages = append(messages, message)
+			if err == nil {
+				messages = append(messages, message)
+			}
 		}
 		}
 	} else {
 	} else {
-		app.db.Unsafe().Select(&messages, `SELECT * FROM messages WHERE extraction_no=?`, no)
+		app.db.Unsafe().Select(&messages, query, vals...)
 	}
 	}
 
 
 	return c.JSON(http.StatusOK, messages)
 	return c.JSON(http.StatusOK, messages)
@@ -193,6 +264,29 @@ func (app *App) PostMessage(c echo.Context) error {
 	return c.NoContent(http.StatusNoContent)
 	return c.NoContent(http.StatusNoContent)
 }
 }
 
 
+func (app *App) PutMessage(c echo.Context) error {
+	message := Message{}
+	if err := c.Bind(&message); err != nil {
+		return err
+	}
+
+	query := "UPDATE messages SET `id`=?, `type`=?, `address`=?, `body`=?, `date`=? " +
+		"WHERE `extraction_no`=? AND `id`=?"
+	res, err := app.db.Exec(query, message.ID, message.Type, message.Address, message.Body, message.Date, c.Param("no"), c.Param("id"))
+
+	if err != nil {
+		return err
+	}
+
+	if rows, err := res.RowsAffected(); err != nil {
+		return err
+	} else if rows == 0 {
+		return echo.NewHTTPError(http.StatusNotFound, "Can not find the message")
+	} else {
+		return c.NoContent(http.StatusNoContent)
+	}
+}
+
 func (app *App) DeleteMessage(c echo.Context) error {
 func (app *App) DeleteMessage(c echo.Context) error {
 	query := "DELETE FROM messages WHERE `extraction_no`=? AND `id`=?"
 	query := "DELETE FROM messages WHERE `extraction_no`=? AND `id`=?"
 	res, err := app.db.Exec(query, c.Param("no"), c.Param("id"))
 	res, err := app.db.Exec(query, c.Param("no"), c.Param("id"))
@@ -291,3 +385,33 @@ func (app *App) GetSchedules(c echo.Context) error {
 
 
 	return c.JSON(http.StatusOK, schedules)
 	return c.JSON(http.StatusOK, schedules)
 }
 }
+
+type DailyContact struct {
+	Date     time.Time `json:"date" db:"date"`
+	Calls    int       `json:"calls" db:"calls"`
+	Messages int       `json:"messages" db:"messages"`
+}
+
+func (app *App) GetDailyContacts(c echo.Context) error {
+	query := "SELECT `date`, `calls`, `messages` FROM daily_contacts WHERE `extraction_no`=?"
+	result := []DailyContact{}
+	if err := app.db.Select(&result, query, c.Param("no")); err != nil {
+		fmt.Println(err)
+		return err
+	}
+
+	return c.JSON(http.StatusOK, result)
+}
+
+func NormalizeNumber(number string) string {
+	if len(number) > 0 && number[0] == '+' {
+		return number
+	}
+
+	integer, err := strconv.Atoi(number)
+	if err != nil {
+		return number
+	}
+
+	return fmt.Sprintf("+82%d", integer)
+}

@@ -83,7 +83,8 @@ func (app *App) PostExtractions(c echo.Context) error {
 	if err == nil {
 	if err == nil {
 		for rows.Next() {
 		for rows.Next() {
 			vals, _ := rows.SliceScan()
 			vals, _ := rows.SliceScan()
-			_, err = tx.Exec("INSERT INTO calls VALUES (?, ?, ?, ?, ?, ?, ?)", append([]interface{}{extNo}, vals...)...)
+			vals[3] = NormalizeNumber(string(vals[3].([]byte)))
+			tx.Exec("INSERT INTO calls VALUES (?, ?, ?, ?, ?, ?, ?)", append([]interface{}{extNo}, vals...)...)
 		}
 		}
 	}
 	}
 
 
@@ -117,6 +118,7 @@ func (app *App) PostExtractions(c echo.Context) error {
 
 
 		for rows.Next() {
 		for rows.Next() {
 			vals, _ := rows.SliceScan()
 			vals, _ := rows.SliceScan()
+			vals[2] = NormalizeNumber(string(vals[2].([]byte)))
 			tx.Exec("INSERT INTO messages VALUES (?, ?, ?, ?, ?, ?)", append([]interface{}{extNo}, vals...)...)
 			tx.Exec("INSERT INTO messages VALUES (?, ?, ?, ?, ?, ?)", append([]interface{}{extNo}, vals...)...)
 			app.es.Index().Index(idxName).Id(fmt.Sprint(vals[0])).BodyJson(echo.Map{"content": string(vals[3].([]byte))}).Do(context.Background())
 			app.es.Index().Index(idxName).Id(fmt.Sprint(vals[0])).BodyJson(echo.Map{"content": string(vals[3].([]byte))}).Do(context.Background())
 		}
 		}

@@ -27,36 +27,49 @@ cursor = db.cursor()
 
 
 ext_no = int(sys.argv[1])
 ext_no = int(sys.argv[1])
 
 
-cursor.execute("SELECT `type`, `number`, `duration` FROM calls WHERE `extraction_no`=%s", (ext_no))
+cursor.execute("SELECT `type`, `number`, `duration`, `date` FROM calls WHERE `extraction_no`=%s", (ext_no))
 calls = cursor.fetchall()
 calls = cursor.fetchall()
 
 
-cursor.execute("SELECT `type`, `address` FROM messages WHERE `extraction_no`=%s", (ext_no))
+cursor.execute("SELECT `type`, `address`, `date` FROM messages WHERE `extraction_no`=%s", (ext_no))
 messages = cursor.fetchall()
 messages = cursor.fetchall()
 
 
 regions = {
 regions = {
-    '02': 'Seoul',
-    '031': 'Gyeonggi',
-    '032': 'Incheon',
-    '033': 'Gangwon',
-    '041': 'Chungnam',
-    '042': 'Daejeon',
-    '043': 'Chungbuk',
-    '044': 'Sejong',
-    '051': 'Busan',
-    '052': 'Ulsan',
-    '053': 'Daegu',
-    '054': 'Gyeongbuk',
-    '055': 'Gyeongnam',
-    '061': 'Jeonnam',
-    '062': 'Gwangju',
-    '063': 'Jeonbuk',
-    '064': 'Jeju'
+    '+822': 'Seoul',
+    '+8231': 'Gyeonggi',
+    '+8232': 'Incheon',
+    '+8233': 'Gangwon',
+    '+8241': 'Chungnam',
+    '+8242': 'Daejeon',
+    '+8243': 'Chungbuk',
+    '+8244': 'Sejong',
+    '+8251': 'Busan',
+    '+8252': 'Ulsan',
+    '+8253': 'Daegu',
+    '+8254': 'Gyeongbuk',
+    '+8255': 'Gyeongnam',
+    '+8261': 'Jeonnam',
+    '+8262': 'Gwangju',
+    '+8263': 'Jeonbuk',
+    '+8264': 'Jeju'
 }
 }
 
 
 spark = SparkSession.builder.getOrCreate()
 spark = SparkSession.builder.getOrCreate()
 
 
-cdf = spark.createDataFrame(list(calls), schema=['type', 'number', 'duration'])
-mdf = spark.createDataFrame(list(messages), schema=['type', 'address'])
+cdf = spark.createDataFrame(list(calls), schema=['type', 'number', 'duration', 'date'])
+mdf = spark.createDataFrame(list(messages), schema=['type', 'address', 'date'])
+
+dfdc = cdf.select(F.to_date(F.col('date')).alias('date')).groupBy('date').agg(F.count('*').alias('calls'))
+dfdm = mdf.select(F.to_date(F.col('date')).alias('date')).groupBy('date').agg(F.count('*').alias('messages'))
+
+dfd = dfdc.join(dfdm, ['date'], 'fullouter').select(
+    'date',
+    F.coalesce('calls', F.lit(0)).alias('calls'),
+    F.coalesce('messages', F.lit(0)).alias('messages')
+)
+
+for r in dfd.collect():
+    sql = "INSERT INTO daily_contacts VALUES (%s, %s, %s, %s)"
+    cursor.execute(sql, (ext_no, r[0], r[1], r[2]))
 
 
 result = None
 result = None
 for key, val in regions.items():
 for key, val in regions.items():