baladiwei
/
ohurlshortener
1
0
Fork 0
Code Issues Pull Requests Projects Releases 9 Wiki Activity

完成页面加载的优化 

1、数据库层:去掉了不必要的 view 设置
2、根据数据库层的改动,对 storage 层进行了相应的修改处理
3、加入了 faker 及大量测试数据的生产方法
4、根据以上的修改对 dashboard.html 页面进行了相应的调整
This commit is contained in:
巴拉迪维 2022-08-19 01:10:23 +08:00
parent cf33c9208e
commit 94eca14838
17 changed files with 191 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
controller/admin_controller.go
View File

@ -40,7 +40,7 @@ func DoLogin(c *gin.Context) {
captchaText := c.PostForm("captcha-text")
captchaId := c.PostForm("captcha-id")
if utils.EemptyString(account) || utils.EemptyString(password) || len(account) < 5 || len(password) < 8 {
if utils.EmptyString(account) || utils.EmptyString(password) || len(account) < 5 || len(password) < 8 {
c.HTML(http.StatusOK, "login.html", gin.H{
"title": "错误 - ohUrlShortener",
"error": "用户名或密码格式错误！",
@ -48,7 +48,7 @@ func DoLogin(c *gin.Context) {
return
}
if utils.EemptyString(captchaText) || utils.EemptyString(captchaId) || len(captchaText) < 6 {
if utils.EmptyString(captchaText) || utils.EmptyString(captchaId) || len(captchaText) < 6 {
c.HTML(http.StatusOK, "login.html", gin.H{
"title": "错误 - ohUrlShortener",
"error": "验证码格式错误!",
@ -108,7 +108,7 @@ func ChangeState(c *gin.Context) {
destUrl := c.PostForm("dest_url")
enable := c.PostForm("enable")
if utils.EemptyString(destUrl) {
if utils.EmptyString(destUrl) {
c.JSON(http.StatusBadRequest, core.ResultJsonError("目标链接不存在！"))
return
}
@ -130,7 +130,7 @@ func ChangeState(c *gin.Context) {
func DeleteShortUrl(c *gin.Context) {
url := c.PostForm("short_url")
if utils.EemptyString(strings.TrimSpace(url)) {
if utils.EmptyString(strings.TrimSpace(url)) {
c.JSON(http.StatusBadRequest, core.ResultJsonError("目标链接不存在！"))
return
}
@ -148,7 +148,7 @@ func GenerateShortUrl(c *gin.Context) {
destUrl := c.PostForm("dest_url")
memo := c.PostForm("memo")
if utils.EemptyString(destUrl) {
if utils.EmptyString(destUrl) {
c.JSON(http.StatusBadRequest, core.ResultJsonError("目标链接不能为空！"))
return
}

14
controller/api_controller.go
View File

@ -26,7 +26,7 @@ import (
func APINewAdmin(ctx *gin.Context) {
account := ctx.PostForm("account")
password := ctx.PostForm("password")
if utils.EemptyString(account) || utils.EemptyString(password) {
if utils.EmptyString(account) || utils.EmptyString(password) {
ctx.JSON(http.StatusBadRequest, core.ResultJsonBadRequest("用户名或密码不能为空"))
return
}
@ -53,7 +53,7 @@ func APIAdminUpdate(ctx *gin.Context) {
account := ctx.Param("account")
password := ctx.PostForm("password")
if utils.EemptyString(account) || utils.EemptyString(password) {
if utils.EmptyString(account) || utils.EmptyString(password) {
ctx.JSON(http.StatusBadRequest, core.ResultJsonBadRequest("用户名或密码不能为空"))
return
}
@ -77,7 +77,7 @@ func APIGenShortUrl(ctx *gin.Context) {
url := ctx.PostForm("dest_url")
memo := ctx.PostForm("memo")
if utils.EemptyString(strings.TrimSpace(url)) {
if utils.EmptyString(strings.TrimSpace(url)) {
ctx.JSON(http.StatusBadRequest, core.ResultJsonBadRequest("dest_url 不能为空"))
return
}
@ -97,13 +97,13 @@ func APIGenShortUrl(ctx *gin.Context) {
// APIUrlInfo Get Short Url Stat Info.
func APIUrlInfo(ctx *gin.Context) {
url := ctx.Param("url")
if utils.EemptyString(strings.TrimSpace(url)) {
if utils.EmptyString(strings.TrimSpace(url)) {
ctx.JSON(http.StatusBadRequest, core.ResultJsonBadRequest("url 不能为空"))
return
}
stat, err := service.GetShortUrlStats(strings.TrimSpace(url))
if utils.EemptyString(strings.TrimSpace(url)) {
if utils.EmptyString(strings.TrimSpace(url)) {
ctx.JSON(http.StatusInternalServerError, core.ResultJsonError(err.Error()))
return
}
@ -115,7 +115,7 @@ func APIUrlInfo(ctx *gin.Context) {
func APIUpdateUrl(ctx *gin.Context) {
url := ctx.Param("url")
enableStr := ctx.PostForm("enable")
if utils.EemptyString(strings.TrimSpace(url)) {
if utils.EmptyString(strings.TrimSpace(url)) {
ctx.JSON(http.StatusBadRequest, core.ResultJsonBadRequest("url 不能为空"))
return
}
@ -137,7 +137,7 @@ func APIUpdateUrl(ctx *gin.Context) {
func APIDeleteUrl(ctx *gin.Context) {
url := ctx.Param("url")
if utils.EemptyString(strings.TrimSpace(url)) {
if utils.EmptyString(strings.TrimSpace(url)) {
ctx.JSON(http.StatusBadRequest, core.ResultJsonBadRequest("url 不能为空"))
return
}

2
controller/handlers.go
View File

@ -31,7 +31,7 @@ const (
func APIAuthHandler() gin.HandlerFunc {
return func(ctx *gin.Context) {
authHeader := ctx.GetHeader(authoriationHeaderKey)
if utils.EemptyString(authHeader) {
if utils.EmptyString(authHeader) {
ctx.AbortWithStatusJSON(http.StatusUnauthorized, core.ResultJsonUnauthorized("Authorization Header is empty"))
return
}

4
controller/urls_controller.go
View File

@ -18,7 +18,7 @@ import (
func ShortUrlDetail(c *gin.Context) {
url := c.Param("url")
if utils.EemptyString(url) {
if utils.EmptyString(url) {
c.HTML(http.StatusNotFound, "error.html", gin.H{
"title": "404 - ohUrlShortener",
"code": http.StatusNotFound,
@ -39,7 +39,7 @@ func ShortUrlDetail(c *gin.Context) {
return
}
if utils.EemptyString(destUrl) {
if utils.EmptyString(destUrl) {
c.HTML(http.StatusNotFound, "error.html", gin.H{
"title": "404 - ohUrlShortener",
"code": http.StatusNotFound,

8
core/short_url.go
View File

@ -11,7 +11,6 @@ package core
import (
"database/sql"
"fmt"
"math/big"
"ohurlshortener/utils"
"reflect"
"time"
@ -31,14 +30,15 @@ func (url ShortUrl) IsEmpty() bool {
}
func GenerateShortLink(initialLink string) (string, error) {
if utils.EemptyString(initialLink) {
if utils.EmptyString(initialLink) {
return "", fmt.Errorf("empty string")
}
urlHash, err := utils.Sha256Of(initialLink)
if err != nil {
return "", err
}
number := new(big.Int).SetBytes(urlHash).Uint64()
str := utils.Base58Encode([]byte(fmt.Sprintf("%d", number)))
// number := new(big.Int).SetBytes(urlHash).Uint64()
// str := utils.Base58Encode([]byte(fmt.Sprintf("%d", number)))
str := utils.Base58Encode(urlHash)
return str[:8], nil
}

5
core/stats.go
View File

@ -31,3 +31,8 @@ type UrlIpCountStats struct {
ShortUrl
ShortUrlStats
}
type StatsSum struct {
Key string `db:"stats_key"`
Value int `db:"stats_value"`
}

6
main.go
View File

@ -31,8 +31,8 @@ import (
const (
ACCESS_LOG_CLEAN_INTERVAL = 1 * time.Minute
WEB_READ_TIMEOUT = 10 * time.Second
WEB_WRITE_TIMEOUT = 10 * time.Second
WEB_READ_TIMEOUT = 30 * time.Second
WEB_WRITE_TIMEOUT = 30 * time.Second
)
var (
@ -83,7 +83,7 @@ func main() {
startAdmin(group, *admin)
} else if strings.EqualFold("portal", strings.TrimSpace(cmdStart)) {
startPortal(group, *portal)
} else if utils.EemptyString(cmdStart) {
} else if utils.EmptyString(cmdStart) {
startPortal(group, *portal)
startAdmin(group, *admin)
} else {

2
service/url_s.go
View File

@ -106,7 +106,7 @@ func GenerateShortUrl(destUrl string, memo string) (string, error) {
}
var nsMemo sql.NullString
if !utils.EemptyString(memo) {
if !utils.EmptyString(memo) {
nsMemo = sql.NullString{Valid: true, String: memo}
}

19
service/url_test.go
View File

@ -9,23 +9,24 @@
package service
import (
"log"
"os"
"testing"
"github.com/bxcodec/faker/v3"
)
func TestGenerateShortUrl(t *testing.T) {
init4Test(t)
if err := StoreAccessLogs(); err != nil {
t.Error(err)
}
res, err := GenerateShortUrl("https://ww2222.ortener", "")
if err != nil {
t.Error(err)
for i := 0; i < 100000; i++ {
url := faker.URL()
_, err := GenerateShortUrl(url, url)
if err != nil {
t.Error(err)
continue
}
}
log.SetOutput(os.Stdout)
log.Println(res)
}

14
storage/access_logs_storage.go
View File

@ -54,13 +54,13 @@ func FindAccessLogsCount(url string, start, end string) (int, int, error) {
UniqueIpCount int `db:"unique_ip_count"`
}
query := `SELECT count(l.id) as total_count, count(distinct(l.ip)) as unique_ip_count FROM public.access_logs l WHERE 1=1 `
if !utils.EemptyString(url) {
if !utils.EmptyString(url) {
query += fmt.Sprintf(` AND l.short_url = '%s'`, url)
}
if !utils.EemptyString(start) {
if !utils.EmptyString(start) {
query += fmt.Sprintf(` AND l.access_time >= to_date('%s','YYYY-MM-DD')`, start)
}
if !utils.EemptyString(end) {
if !utils.EmptyString(end) {
query += fmt.Sprintf(` AND l.access_time < to_date('%s','YYYY-MM-DD')`, end)
}
var count LogsCount
@ -71,13 +71,13 @@ func FindAllAccessLogs(url string, start, end string, page, size int) ([]core.Ac
found := []core.AccessLog{}
offset := (page - 1) * size
query := `SELECT * FROM public.access_logs l WHERE 1=1 `
if !utils.EemptyString(url) {
if !utils.EmptyString(url) {
query += fmt.Sprintf(` AND l.short_url = '%s'`, url)
}
if !utils.EemptyString(start) {
if !utils.EmptyString(start) {
query += fmt.Sprintf(` AND l.access_time >= to_date('%s','YYYY-MM-DD')`, start)
}
if !utils.EemptyString(end) {
if !utils.EmptyString(end) {
query += fmt.Sprintf(` AND l.access_time < to_date('%s','YYYY-MM-DD')`, end)
}
@ -88,7 +88,7 @@ func FindAllAccessLogs(url string, start, end string, page, size int) ([]core.Ac
func FindAllAccessLogsByUrl(url string) ([]core.AccessLog, error) {
found := []core.AccessLog{}
query := "SELECT * FROM public.access_logs l ORDER BY l.id DESC"
if !utils.EemptyString(url) {
if !utils.EmptyString(url) {
query := "SELECT * FROM public.access_logs l WHERE l.short_url = $1 ORDER BY l.id DESC"
err := DbSelect(query, &found, url)
return found, err

16
storage/access_logs_storage_test.go
View File

@ -2,6 +2,7 @@ package storage
import (
"database/sql"
"math/rand"
"ohurlshortener/core"
"testing"
"time"
@ -9,13 +10,22 @@ import (
"github.com/bxcodec/faker/v3"
)
func randomTime() time.Time {
min := time.Date(2022, 1, 0, 0, 0, 0, 0, time.UTC).Unix()
max := time.Now().Unix()
delta := max - min
sec := rand.Int63n(delta) + min
return time.Unix(sec, 0)
}
func TestNewAccessLog(t *testing.T) {
init4Test(t)
var logs []core.AccessLog
for i := 0; i < 15000; i++ {
log := core.AccessLog{ShortUrl: "AC7VgPE9",
for i := 0; i < 100000; i++ {
log := core.AccessLog{ShortUrl: "AvTkHZP7",
Ip: sql.NullString{Valid: true, String: faker.IPv4()},
AccessTime: time.Now(),
AccessTime: randomTime(),
UserAgent: sql.NullString{Valid: true, String: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.80 Safari/537.36"}}
logs = append(logs, log)
}

44
storage/stats_storage.go
View File

@ -15,17 +15,43 @@ func GetUrlStats(url string) (core.ShortUrlStats, error) {
func GetUrlCount() (int, error) {
var result int
query := `SELECT count(l.id) FROM public.short_urls l`
// query := `SELECT n_live_tup AS estimate_rows FROM pg_stat_all_tables WHERE relname = 'short_urls'`
return result, DbGet(query, &result)
}
func GetSumOfUrlStats() (core.ShortUrlStats, error) {
query := `SELECT * FROM public.sum_url_ip_count_stats`
found := core.ShortUrlStats{}
return found, DbGet(query, &found)
query := `SELECT * FROM public.stats_sum`
result := core.ShortUrlStats{}
data := []core.StatsSum{}
err := DbSelect(query, &data)
if err != nil {
return result, err
}
for _, v := range data {
switch v.Key {
case "today_count":
result.TodayCount = v.Value
case "yesterday_count":
result.YesterdayCount = v.Value
case "last_7_days_count":
result.Last7DaysCount = v.Value
case "monthly_count":
result.MonthlyCount = v.Value
case "d_today_count":
result.DistinctTodayCount = v.Value
case "d_yesterday_count":
result.DistinctYesterdayCount = v.Value
case "d_last_7_days_count":
result.DistinctLast7DaysCount = v.Value
case "d_monthly_count":
result.DistinctMonthlyCount = v.Value
}
}
return result, nil
}
func GetTop25() ([]core.Top25Url, error) {
query := `SELECT * FROM public.total_count_top25`
query := `SELECT u.*,s.today_count AS today_count,s.d_today_count AS d_today_count FROM public.short_urls u , public.stats_top25 s WHERE u.short_url = s.short_url`
found := []core.Top25Url{}
return found, DbSelect(query, &found)
}
@ -33,11 +59,11 @@ func GetTop25() ([]core.Top25Url, error) {
func FindPagedUrlIpCountStats(url string, page int, size int) ([]core.UrlIpCountStats, error) {
found := []core.UrlIpCountStats{}
offset := (page - 1) * size
query := `SELECT s.*, u.id,u.dest_url,u.created_at,u.is_valid,u.memo
FROM public.url_ip_count_stats s , public.short_urls u WHERE s.short_url = u.short_url ORDER BY u.created_at DESC LIMIT $1 OFFSET $2`
if !utils.EemptyString(url) {
query := `SELECT s.*, u.id,u.dest_url,u.created_at,u.is_valid,u.memo
FROM public.url_ip_count_stats s , public.short_urls u WHERE s.short_url = u.short_url AND u.short_url = $1 ORDER BY u.created_at DESC LIMIT $2 OFFSET $3`
query := `SELECT s.*,u.id,u.dest_url,u.created_at,u.is_valid,u.memo
FROM public.stats_ip_sum s , public.short_urls u WHERE u.short_url = s.short_url ORDER BY u.created_at DESC LIMIT $1 OFFSET $2`
if !utils.EmptyString(url) {
query := `SELECT s.*,u.id,u.dest_url,u.created_at,u.is_valid,u.memo
FROM public.stats_ip_sum s , public.short_urls u WHERE u.short_url = s.short_url AND u.short_url = $1 ORDER BY u.created_at DESC LIMIT $2 OFFSET $3`
var foundUrl core.UrlIpCountStats
err := DbGet(query, &foundUrl, url, size, offset)
if !foundUrl.IsEmpty() {

4
storage/urls_storage.go
View File

@ -14,7 +14,7 @@ import (
"ohurlshortener/utils"
)
var Max_Insert_Count = 5000
var Max_Insert_Count = 1000
func UpdateShortUrl(shortUrl core.ShortUrl) error {
query := `UPDATE public.short_urls SET short_url = :short_url, dest_url = :dest_url, is_valid = :is_valid, memo = :memo WHERE id = :id`
@ -50,7 +50,7 @@ func FindPagedShortUrls(url string, page int, size int) ([]core.ShortUrl, error)
found := []core.ShortUrl{}
offset := (page - 1) * size
query := "SELECT * FROM public.short_urls u ORDER BY u.id DESC LIMIT $1 OFFSET $2"
if !utils.EemptyString(url) {
if !utils.EmptyString(url) {
query := "SELECT * FROM public.short_urls u WHERE u.short_url = $1 ORDER BY u.id DESC LIMIT $2 OFFSET $3"
var foundUrl core.ShortUrl
err := DbGet(query, &foundUrl, url, size, offset)

18
storage/urls_storage_test.go
View File

@ -1,10 +1,28 @@
package storage
import (
"database/sql"
"ohurlshortener/core"
"testing"
"time"
"github.com/bxcodec/faker/v3"
)
func TestInsertShortUrls(t *testing.T) {
init4Test(t)
for i := 0; i < 10000; i++ {
destUrl := faker.URL()
shortUrl, _ := core.GenerateShortLink(destUrl)
url := core.ShortUrl{DestUrl: destUrl, ShortUrl: shortUrl, CreatedAt: time.Now(), Valid: true, Memo: sql.NullString{String: destUrl, Valid: true}}
err := InsertShortUrl(url)
if err != nil {
t.Error(err)
}
}
}
func TestDeleteShortUrlWithAccessLogs(t *testing.T) {
init4Test(t)

19
structure.sql
View File

@ -68,16 +68,15 @@ FROM public.short_urls u
GROUP BY u.short_url;
CREATE VIEW public.sum_url_ip_count_stats AS
SELECT
COUNT(l.ip) AS today_count,
COUNT(DISTINCT(l.ip)) AS d_today_count
FROM public.access_logs l
WHERE date(l.access_time) = date(NOW());
SELECT
COUNT(l.ip) AS today_count,
COUNT(DISTINCT(l.ip)) AS d_today_count
FROM public.access_logs l
WHERE date(l.access_time) = date(NOW());
CREATE VIEW public.total_count_top25 AS
SELECT s.*, u.id,u.dest_url,u.created_at,u.is_valid,u.memo
FROM public.url_ip_count_stats s, public.short_urls u
WHERE u.short_url = s.short_url
ORDER BY s.today_count DESC
LIMIT 25;
SELECT u.*,tmp_logs.today_count, tmp_logs.d_today_count FROM public.short_urls u,(
SELECT l.short_url,count(l.ip) AS today_count ,COUNT(DISTINCT(l.ip)) AS d_today_count
FROM public.access_logs l WHERE date(l.access_time) = date(NOW()) GROUP BY l.short_url ORDER BY today_count DESC LIMIT 25) AS tmp_logs
WHERE u.short_url = tmp_logs.short_url;

67
templates/admin/dashboard.html
View File

@ -55,6 +55,73 @@
</div>
</div><!--end of column-->
</div><!--end of grid-->
<div class="ui center aligned grid stackable padded">
<div class="three wide computer eight wide tablet sixteen wide mobile column">
<div class="ui fluid card">
<div class="content">
<div class="header">
<div class="ui red header">{{.stats.YesterdayCount}}</div>
</div>
<div class="meta">昨日点击量</div>
<div class="description">所有链接昨日产生的总点击量</div>
</div>
<div class="extra content">
<div class="ui two buttons">
<a href="/admin/stats" target="_self" class="ui red button">查看详情</a>
</div>
</div>
</div>
</div><!--end of column-->
<div class="three wide computer eight wide tablet sixteen wide mobile column">
<div class="ui fluid card">
<div class="content">
<div class="header">
<div class="ui header green">{{.stats.DistinctYesterdayCount}}</div>
</div>
<div class="meta">昨日独立IP数</div>
<div class="description">所有短链接昨日点击的总IP数</div>
</div>
<div class="extra content">
<div class="ui two buttons">
<a href="/admin/stats" target="_self" class="ui green button">查看详情</a>
</div>
</div>
</div>
</div><!--end of column-->
<div class="three wide computer eight wide tablet sixteen wide mobile column">
<div class="ui fluid card">
<div class="content">
<div class="header">
<div class="ui teal header">{{.stats.Last7DaysCount}}</div>
</div>
<div class="meta">过去七日点击量</div>
<div class="description">所有短链接过去七日总点击量</div>
</div>
<div class="extra content">
<div class="ui two buttons">
<a href="/admin/stats" target="_self" class="ui teal button">查看详情</a>
</div>
</div>
</div>
</div><!--end of column-->
<div class="three wide computer eight wide tablet sixteen wide mobile column">
<div class="ui fluid card">
<div class="content">
<div class="header">
<div class="ui yellow header">{{.stats.DistinctLast7DaysCount}}</div>
</div>
<div class="meta">过去七日独立IP数</div>
<div class="description">所有短链接过去七日点击的总IP数</div>
</div>
<div class="extra content">
<div class="ui two buttons">
<a href="/admin/stats" target="_self" class="ui yellow button">查看详情</a>
</div>
</div>
</div>
</div><!--end of column-->
</div><!--end of grid-->
<div class="ui grid stackable padded">
<div class="column">

4
utils/utils.go
View File

@ -34,13 +34,13 @@ func PrintOnError(message string, err error) {
}
func RaiseError(message string) error {
if !EemptyString(message) {
if !EmptyString(message) {
return fmt.Errorf(message)
}
return nil
}
func EemptyString(str string) bool {
func EmptyString(str string) bool {
str = strings.TrimSpace(str)
return strings.EqualFold(str, "")
}