接入百度统计部分数据,按天存储

This commit is contained in:
xxq250 2024-01-15 11:23:59 +08:00
parent bf3f25972e
commit e2374676c3
11 changed files with 522 additions and 3 deletions

View File

@ -28,6 +28,41 @@ class Admins::DashboardsController < Admins::BaseController
@day_new_project_count = Project.where(created_on: today).count
@weekly_new_project_count = Project.where(created_on: current_week).count
@month_new_project_count = Project.where(created_on: current_month).count
# 总的平台用户数
# 总的平台项目数
# 总的平台组织数
# 总的平台Issue数、评论数、PR数、Commit数
@user_count = User.count
@project_count = Project.count
@organization_count = Organization.count
@issue_count = Issue.count
@comment_count = Journal.count
@pr_count = PullRequest.count
@commit_count = CommitLog.count
@subject_name = ["用户数", "项目数", "组织数", "Issue数", "Issue评论数", "PR数", "Commit数"]
@subject_icon = ["fa-user","fa-git", "fa-sitemap", "fa-warning", "fa-comments", "fa-share-alt", "fa-upload"]
@subject_data = [@user_count, @project_count, @organization_count, @issue_count, @comment_count, @pr_count, @commit_count]
tongji_service = Baidu::TongjiService.new
@access_token = tongji_service.access_token
Rails.logger.info "baidu_tongji_auth access_token ===== #{@access_token}"
# @overview_data = tongji_service.api_overview
last_date = DailyPlatformStatistic.order(:date).last
start_date = last_date.date
end_date = Time.now
if @access_token.present?
@overview_data = tongji_service.overview_batch_add(start_date, end_date)
tongji_service.source_from_batch_add(start_date, end_date)
end
@current_week_statistic = DailyPlatformStatistic.where(date: current_week)
@pre_week_statistic = DailyPlatformStatistic.where(date: pre_week)
end
def month_active_user
@ -42,6 +77,19 @@ class Admins::DashboardsController < Admins::BaseController
render_ok(data: data)
end
def baidu_tongji
tongji_service = Baidu::TongjiService.new
redirect_to tongji_service.code_url
end
def baidu_tongji_auth
if params[:code].present?
tongji_service = Baidu::TongjiService.new
tongji_service.get_access_token(params[:code])
end
redirect_to "/admins/"
end
def evaluate
names = []
data = []
@ -63,8 +111,12 @@ class Admins::DashboardsController < Admins::BaseController
Time.now.beginning_of_day..Time.now.end_of_day
end
def current_week
def pre_7_days
7.days.ago.end_of_day..Time.now.end_of_day
end
def current_week
Time.now.beginning_of_week..Time.now.end_of_day
end
def current_month
@ -72,6 +124,7 @@ class Admins::DashboardsController < Admins::BaseController
end
def pre_week
14.days.ago.end_of_day..7.days.ago.end_of_day
# 14.days.ago.end_of_day..7.days.ago.end_of_day
Time.now.prev_week..Time.now.prev_week.end_of_week
end
end

View File

@ -0,0 +1,44 @@
# 按天获取百度统计数据pv访问ip和来源分类占比
# 其他统计:前一周用户留存率
class DailyPlatformStatisticsJob < ApplicationJob
queue_as :default
def perform(*args)
Rails.logger.info("*********开始统计*********")
tongji_service = Baidu::TongjiService.new
access_token = tongji_service.access_token
Rails.logger.info "job baidu_tongji_auth access_token ===== #{access_token}"
ActiveJob::Base.logger.info "job baidu_tongji_auth access_token ===== #{access_token}"
# 从最后一个记录日期开始,如果遗漏日期数据可以补充数据
last_date = DailyPlatformStatistic.order(:date).last
start_date = last_date.date
end_date = Time.now
if access_token.present?
tongji_service.overview_batch_add(start_date, end_date)
# 本周访问来源占比,每天记录一次,如果遗漏日期数据可以补充数据
tongji_service.source_from_batch_add(start_date, end_date)
end
# 周用户留存率
pre_week_user_ids = User.where(created_on: pre_week).pluck(:id).uniq
weekly_keep_user_count = User.where(id: pre_week_user_ids).where(last_login_on: current_week).count
weekly_keep_rate = format("%.2f", pre_week_user_ids.size > 0 ? weekly_keep_user_count.to_f / pre_week_user_ids.size : 0)
job_date = 1.days.ago
daily_statistic = DailyPlatformStatistic.find_or_initialize_by(date: job_date)
daily_statistic.weekly_keep_rate = weekly_keep_rate
daily_statistic.save
end
private
def current_week
Time.now.beginning_of_week..Time.now.end_of_day
end
def pre_week
# 7.days.ago.beginning_of_week..7.days.ago.beginning_of_week.end_of_week
Time.now.prev_week..Time.now.prev_week.end_of_week
end
end

View File

@ -0,0 +1,24 @@
# == Schema Information
#
# Table name: daily_platform_statistics
#
# id :integer not null, primary key
# date :date
# pv :integer default("0")
# visitor :integer default("0")
# ip :integer default("0")
# weekly_keep_rate :float(24) default("0")
# source_through :float(24) default("0")
# source_link :float(24) default("0")
# source_search :float(24) default("0")
# source_custom :float(24) default("0")
# created_at :datetime not null
# updated_at :datetime not null
#
# Indexes
#
# index_daily_platform_statistics_on_date (date) UNIQUE
#
class DailyPlatformStatistic < ApplicationRecord
end

View File

@ -0,0 +1,221 @@
module Baidu
class TongjiService < ApplicationService
attr_reader :client_id, :client_secret, :site_id
# login、code、password、password_confirmation
def initialize
@client_id = "6dMO2kqKUaMZkBrMaUMxQSNAT49v0Mjq"
@client_secret = "qvWqF33AOmGs1tPCgsROvis9EQCuNmd3"
@site_id = 18657013
end
def call
end
def init_overview_data_by(start_date = nil, end_date = nil)
start_date = Time.now.prev_year.beginning_of_year if start_date.nil?
end_date = Time.now
Rails.logger.info("*********开始百度统计-概览:#{start_date}-#{end_date}*********")
sql_connection = ActiveRecord::Base.connection
sql_connection.begin_db_transaction
# 如果存在数据 先清空
# sql_connection.execute("delete from daily_platform_statistics where date between '#{start_date}' and '#{end_date}'")
multiple_days_data = overview_multiple_days_data(start_date, end_date)
if multiple_days_data.present?
sql = "replace into daily_platform_statistics (date,pv,visitor,ip,created_at,updated_at) values #{multiple_days_data.join(",")}"
sql_connection.execute(sql)
end
sql_connection.commit_db_transaction
Rails.logger.info("*********结束百度统计-概览:#{start_date}-#{end_date}*********")
end
def init_source_from_data_by(start_date = nil, end_date = nil)
start_date = Time.now.prev_year.beginning_of_year if start_date.nil?
end_date = Time.now
Rails.logger.info("*********开始百度统计-来源:#{start_date}-#{end_date}*********")
source_from_batch_add(start_date, end_date)
Rails.logger.info("*********结束百度统计-来源:#{start_date}-#{end_date}*********")
end
# 按日期获取来源数据
def source_from_batch_add(start_date,end_date)
# 补充更新开始时间的当天数据
source_from_by_date(start_date)
diff_days(start_date, end_date).times.each do |t|
new_start_date = start_date + (t + 1).days
source_from_by_date(new_start_date)
end
# 补充更新最后时间一天数据
source_from_by_date(end_date)
end
# 按天获取来源数据
def source_from_by_date(start_date)
return [] unless access_token.present? && start_date.present?
source_from_data = api("source/all/a", start_date, start_date, "pv_count,visitor_count,ip_count")
source_from = []
source_from_data['items'][1].each_with_index do |source, index|
source_from.push(((source[0].to_f / source_from_data['sum'][0][0].to_f) * 100).round(2))
end
daily_statistic = DailyPlatformStatistic.find_or_initialize_by(date: start_date)
daily_statistic.source_through = source_from[0]
daily_statistic.source_link = source_from[1]
daily_statistic.source_search = source_from[2]
daily_statistic.source_custom = source_from[3]
daily_statistic.save
end
def diff_days(start_date, end_date)
(end_date.beginning_of_day.to_i - start_date.beginning_of_day.to_i) / (24 * 3600)
end
def overview_batch_add(start_date, end_date)
return [] unless access_token.present? && start_date.present? && end_date.present?
start_date = Time.now - 1.days if start_date.strftime("%Y%m%d") == end_date.strftime("%Y%m%d")
overview_data = api("overview/getTimeTrendRpt", start_date, end_date, "pv_count,visitor_count,ip_count")
overview_data['items'][0].each_with_index do |date, index|
pv = overview_data['items'][1][index][0]
visitor = overview_data['items'][1][index][1]
ip = overview_data['items'][1][index][2]
job_date = date[0].to_s.gsub("/", "-")
daily_statistic = DailyPlatformStatistic.find_or_initialize_by(date: job_date)
daily_statistic.date = job_date
daily_statistic.pv = pv
daily_statistic.visitor = visitor
daily_statistic.ip = ip
daily_statistic.save
end
overview_data
end
def overview_multiple_days_data(start_date, end_date)
return [] unless access_token.present? && start_date.present? && end_date.present?
overview_data = api("overview/getTimeTrendRpt", start_date, end_date, "pv_count,visitor_count,ip_count")
data = []
created_at = Time.now.strftime("%Y-%m-%d 00:00:00")
overview_data['items'][0].each_with_index do |date, index|
pv = overview_data['items'][1][index][0]
visitor = overview_data['items'][1][index][1]
ip = overview_data['items'][1][index][2]
data.push("('#{date[0].to_s.gsub("/", "-")}', #{pv.to_s.gsub("--","0")}, #{visitor.to_s.gsub("--","0")}, #{ip.to_s.gsub("--","0")},\"#{created_at}\",\"#{created_at}\")")
end
data
end
def code_url
"http://openapi.baidu.com/oauth/2.0/authorize?response_type=code&client_id=#{client_id}&redirect_uri=oob&scope=basic&display=popup"
end
def oauth_url(code)
"http://openapi.baidu.com/oauth/2.0/token?grant_type=authorization_code&code=#{code}&client_id=#{client_id}&client_secret=#{client_secret}&redirect_uri=oob"
end
def get_access_token(code)
uri = URI.parse(oauth_url(code))
response = Net::HTTP.get_response(uri)
Rails.logger.info "baidu_tongji_auth response.body ===== #{response.body}"
if response.code.to_i == 200
data = JSON.parse(response.body)
access_token = data['access_token']
refresh_token = data['refresh_token']
expires_in = data['expires_in']
if access_token.present?
Rails.cache.write("baidu_tongji_auth/access_token", access_token, expires_in: expires_in)
Rails.cache.write("baidu_tongji_auth/refresh_token", refresh_token, expires_in: 1.year)
end
end
end
def refresh_access_token
url = "http://openapi.baidu.com/oauth/2.0/token?grant_type=refresh_token&refresh_token=#{refresh_token}&client_id=#{client_id}&client_secret=#{client_secret}"
uri = URI.parse(url)
response = Net::HTTP.get_response(uri)
Rails.logger.info "baidu_tongji_auth response.body ===== #{response.body}"
if response.code.to_i == 200
data = JSON.parse(response.body)
access_token = data['access_token']
refresh_token = data['refresh_token']
expires_in = data['expires_in']
if access_token.present?
Rails.cache.write("baidu_tongji_auth/access_token", access_token, expires_in: expires_in)
Rails.cache.write("baidu_tongji_auth/refresh_token", refresh_token, expires_in: 1.year)
end
end
end
def access_token
access_token = Rails.cache.read("baidu_tongji_auth/access_token")
if access_token.blank?
refresh_access_token
access_token = Rails.cache.read("baidu_tongji_auth/access_token")
end
access_token
end
def refresh_token
refresh_token = Rails.cache.read("baidu_tongji_auth/refresh_token")
# 如果刷新token失效access_token也重置
if refresh_token.blank?
Rails.cache.delete("baidu_tongji_auth/access_token")
end
refresh_token
end
# 网站概况(趋势数据)
def api_overview
start_date = Time.now.beginning_of_week
end_date = Time.now
start_date = Time.now - 1.days if start_date.strftime("%Y%m%d") == end_date.strftime("%Y%m%d")
api("overview/getTimeTrendRpt", start_date, end_date, "pv_count,visitor_count,ip_count")
end
# 网站概况(来源网站、搜索词、入口页面、受访页面)
def api_overview_getCommonTrackRpt
start_date = Time.now.beginning_of_week
end_date = Time.now
api("overview/getCommonTrackRpt", start_date, end_date, "pv_count")
end
# 全部来源
def source_from
start_date = Time.now.beginning_of_week
end_date = Time.now
api("source/all/a", start_date, end_date, "pv_count,visitor_count,ip_count")
end
def api(api_method, start_date, end_date, metrics = nil)
start_date_fmt = start_date.strftime("%Y%m%d")
end_date_fmt = end_date.strftime("%Y%m%d")
api_url = "https://openapi.baidu.com/rest/2.0/tongji/report/getData?access_token=#{access_token}&site_id=#{site_id}&method=#{api_method}&start_date=#{start_date_fmt}&end_date=#{end_date_fmt}&metrics=#{metrics}"
data = url_http_post(api_url, {})
data['result']
end
def url_http_post(api_url, params)
Rails.logger.info "api_url==#{api_url}"
uri = URI.parse(api_url)
http = Net::HTTP.new uri.host, uri.port
http.open_timeout = 60
http.read_timeout = 60
if uri.scheme == 'https'
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
http.use_ssl = true
end
begin
request = Net::HTTP::Post.new(uri)
request.set_form_data(params) if params.present?
request['Content-Type'] = 'application/json;charset=utf-8'
# request['Content-Type'] = 'application/x-www-form-urlencoded;charset=utf-8'
response = http.start { |http| http.request(request) }
Rails.logger.info "api response.body==#{response.body}"
JSON.parse response.body
rescue => err
Rails.logger.error("#############api_url:#{api_url},error:#{err.message.size}")
# Rails.logger.error("#############api_url:#{api_url},error:#{err.message}")
return {}
end
end
end
end

View File

@ -0,0 +1,50 @@
<div>
数据来源百度统计,本周 [<%= @current_week_statistic.first&.date %> / <%= @current_week_statistic.last&.date %>]
</div>
<table class="table table-hover text-center subject-list-table">
<thead class="thead-light">
<tr>
<th width="20%">日期</th>
<th width="15%">访问量</th>
<th width="15%">访客数</th>
<th width="15%">IP数</th>
<th width="15%">直接访问占比</th>
<th width="15%">外部链接占比</th>
<th width="15%">搜索引擎占比</th>
<th width="20%">自定义</th>
</tr>
</thead>
<tbody>
<% @current_week_statistic.each_with_index do |week, index| %>
<tr class="">
<td><%= week.date %> </td>
<td><%= week.pv %></td>
<td><%= week.visitor %></td>
<td><%= week.ip %></td>
<td><%= week.source_through %>%</td>
<td><%= week.source_link %>%</td>
<td><%= week.source_search %>%</td>
<td><%= week.source_custom.to_f %>%</td>
</td>
</tr>
<% end %>
<!-- <tr class="">-->
<!-- <td><span style="color: #ff8200">合计</span></td>-->
<!-- <td><span style="color: #ff8200"><%#= @current_week_statistic.sum(:pv) %></span></td>-->
<!-- <td><span style="color: #ff8200"><%#= @current_week_statistic.sum(:visitor) %></span></td>-->
<!-- <td><span style="color: #ff8200"><%#= @current_week_statistic.sum(:ip) %></span></td>-->
<!-- </td>-->
<!-- </tr>-->
</tbody>
</table>
<% unless @access_token.present? && @overview_data.present? %>
<div>
<a href="/admins/baidu_tongji" target="_blank">1.百度统计需人工授权,点击去授权</a>
</div>
<div>
<a href="/admins/baidu_tongji_auth" onclick="this.href = '/admins/baidu_tongji_auth?code=' + document.getElementById('auth_code').value">2.获取百度统计授权码</a>
<input name="code" id="auth_code" style="width: 380px" placeholder="请输入1.百度统计返回的code后点击2.获取百度统计Token"/>
</div>
<% end %>

View File

@ -0,0 +1,67 @@
<% if @access_token.present? && @overview_data.present? %>
<div>
数据来源百度统计,本周 <%= @overview_data['timeSpan'] %>
</div>
<table class="table table-hover text-center subject-list-table">
<thead class="thead-light">
<tr>
<th width="20%">日期</th>
<th width="15%">访问量</th>
<th width="15%">访客数</th>
<th width="15%">IP数</th>
</tr>
</thead>
<tbody>
<% pv_count = [] %>
<% visitor_count = [] %>
<% ip_count = [] %>
<% @overview_data['items'][0].each_with_index do |date, index| %>
<% pv = @overview_data['items'][1][index][0] %>
<% visitor = @overview_data['items'][1][index][1] %>
<% ip = @overview_data['items'][1][index][2] %>
<tr class="">
<td><%= date[0] %> </td>
<td><%= pv %></td>
<td><%= visitor %></td>
<td><%= ip %></td>
</td>
</tr>
<% pv_count.push(pv) %>
<% visitor_count.push(visitor) %>
<% ip_count.push(ip) %>
<% end %>
<tr class="">
<td><span style="color: #ff8200">合计</span></td>
<td><span style="color: #ff8200"><%= pv_count %></span></td>
<td><span style="color: #ff8200"><%= visitor_count %></span></td>
<td><span style="color: #ff8200"><%= ip_count %></span></td>
</td>
</tr>
</tbody>
</table>
<table class="table table-hover text-center subject-list-table">
<thead class="thead-light">
<tr>
<th width="15%">直接访问占比</th>
<th width="15%">外部链接占比</th>
<th width="15%">搜索引擎占比</th>
<th width="20%">自定义</th>
</tr>
</thead>
<tbody>
<tr class="">
<% @source_from_data['items'][1].each_with_index do |source, index| %>
<td><%= ((source[0].to_f / @source_from_data['sum'][0][0].to_f) * 100).round(2).to_s %>%</td>
<% end %>
</tr>
</tbody>
</table>
<% else %>
<div>
<a href="/admins/baidu_tongji" target="_blank">1.百度统计需人工授权,点击去授权</a>
</div>
<div>
<a href="/admins/baidu_tongji_auth" onclick="this.href = '/admins/baidu_tongji_auth?code=' + document.getElementById('auth_code').value">2.获取百度统计授权码</a>
<input name="code" id="auth_code" style="width: 380px" placeholder="请输入1.百度统计返回的code后点击2.获取百度统计Token"/>
</div>
<% end %>

View File

@ -1,6 +1,35 @@
<% define_admin_breadcrumbs do %>
<% add_admin_breadcrumb('概览', admins_path) %>
<% end %>
<div class="header bg-gradient-primary pb-8 pt-md-8">
<div class="container-fluid">
<div class="header-body">
<!-- Card stats -->
<div class="row">
<%@subject_name.each_with_index do |subject, index| %>
<div class="col-xl-3 col-lg-6" style="padding-bottom: 15px;">
<div class="card card-stats mb-4 mb-xl-0">
<div class="card-body">
<div class="row">
<div class="col">
<h5 class="card-title text-uppercase11 text-muted mb-0"><%=subject %></h5>
<span class="h2 font-weight-bold mb-0"><%= @subject_data[index] %></span>
</div>
<div class="col-auto">
<div class="icon icon-shape rounded-circle shadow">
<i class="fa <%=@subject_icon[index] %>"></i>
</div>
</div>
</div>
</div>
</div>
</div>
<% end %>
</div>
</div>
</div>
</div>
<div class="box admin-list-container project-language-list-container">
<table class="table table-hover text-center subject-list-table">
@ -53,6 +82,7 @@
</tr>
</tbody>
</table>
<%= render partial: 'admins/dashboards/baidu_tongji' %>
</div>
<div id="project-language-modals">
</div>

View File

@ -795,6 +795,8 @@ Rails.application.routes.draw do
namespace :admins do
mount Sidekiq::Web => '/sidekiq'
get '/', to: 'dashboards#index'
get '/baidu_tongji', to: 'dashboards#baidu_tongji'
get '/baidu_tongji_auth', to: 'dashboards#baidu_tongji_auth'
namespace :topic do
resources :activity_forums
resources :banners

View File

@ -11,4 +11,9 @@ delay_expired_issue:
create_daily_project_statistics:
cron: "0 1 * * *"
class: "DailyProjectStatisticsJob"
queue: cache
queue: cache
daily_platform_statistics:
cron: "0 1 * * *"
class: "DailyPlatformStatisticsJob"
queue: default

View File

@ -0,0 +1,18 @@
class CreateDailyPlatformStatistics < ActiveRecord::Migration[5.2]
def change
create_table :daily_platform_statistics do |t|
t.date :date
t.index :date, unique: true
t.bigint :pv, default: 0
t.bigint :visitor, default: 0
t.bigint :ip, default: 0
t.float :weekly_keep_rate, default: 0
t.float :source_through, default: 0
t.float :source_link, default: 0
t.float :source_search, default: 0
t.float :source_custom, default: 0
t.timestamps
end
end
end

View File

@ -0,0 +1,5 @@
require 'rails_helper'
RSpec.describe DailyPlatformStatistic, type: :model do
pending "add some examples to (or delete) #{__FILE__}"
end