Gitlink
/
forgeplus
40
1
Fork 57
Code Issues 1 Pull Requests 1 Packages Releases 15 Wiki Activity

迁移数据的修改

This commit is contained in:
sylor_huang@126.com 2020-04-01 11:07:01 +08:00
parent ea4cfab819
commit cc242a2765
5 changed files with 113 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/forms/gitea/user_form.rb
View File

@ -10,7 +10,7 @@ class Gitea::UserForm
validates :password, presence: true
validate :check_username, :check_email
# validate :check_username, :check_email
attr_reader :record

138
app/jobs/sync_forge_job.rb
View File

@ -2,12 +2,17 @@ class SyncForgeJob < ApplicationJob
queue_as :default
def perform(sync_params)
@change_issue_ids = []
@change_journal_ids =[]
@change_watcher_ids = []
@change_pr_ids = []
@change_praise_trend_ids = []
@change_version_ids = []
Rails.logger.info("#######______sync__start__########")
sync_params = ActiveSupport::JSON.decode(sync_params)
all_target_params = sync_params["target_params"]
roles_params = sync_params["roles"]
user_params = sync_params["user_params"]["user_params"]
owner_extension_params = sync_params["user_params"]["user_extension_params"]
@ -19,8 +24,6 @@ class SyncForgeJob < ApplicationJob
if new_user.present?
ActiveRecord::Base.transaction do
begin
sync_roles(roles_params, platform)
if all_target_params.present?
all_target_params.each do |project|
target_params = {
@ -38,13 +41,6 @@ class SyncForgeJob < ApplicationJob
sync_projects(new_user, user_old_id,target_params, platform)
end
end
Issue.select(:id, :assigned_to_id).where(assigned_to_id: user_old_id)&.update_all(assigned_to_id: new_user.id)
Issue.select(:id, :author_id).where(author_id: user_old_id)&.update_all(author_id: new_user.id)
Journal.select(:id,:user_id).where(user_id: user_old_id)&.update_all(user_id: new_user.id)
Watcher.select(:id,:user_id).where(user_id: user_old_id)&.update_all(user_id: new_user.id)
ProjectTrend.select(:id,:user_id).where(user_id: user_old_id)&.update_all(user_id: new_user.id)
PullRequest.select(:id,:user_id).where(user_id: user_old_id)&.update_all(user_id: new_user.id)
Version.select(:id,:user_id).where(user_id: user_old_id)&.update_all(user_id: new_user.id)
rescue Exception => e
Rails.logger.info("#######_______forge_new_user_sync_failed___#########{e}")
raise ActiveRecord::Rollback
@ -57,7 +53,7 @@ class SyncForgeJob < ApplicationJob
end
def old_version_source
%w(trustie)
%w(trustie military)
end
private
@ -66,28 +62,21 @@ class SyncForgeJob < ApplicationJob
ActiveRecord::Base.transaction do
begin
Rails.logger.info("#######______sync_user_start__########")
keys_other_delete = %w(id created_at updated_at user_id)
keys_to_delete = %w(id created_on updated_on platform)
keys_other_delete = %w(id created_at updated_at user_id province)
keys_to_delete = %w(id created_on updated_on platform admin_role enterprise_certification)
owner_params = owner_params["user"] if old_version_source.include?(platform) #trustie上需要
owner_params = owner_params&.except!(*keys_to_delete)
user_password = random_password
new_user = []
if owner_params.present?
if User.exists?(login: owner_params["login"])
new_user = User.find_by(login: owner_params["login"])
elsif User.exists?(mail: owner_params["mail"])
new_user = User.find_by(mail: owner_params["mail"])
else
new_user = User.new(owner_params.merge(platform: platform))
interactor = Gitea::RegisterInteractor.call({username: owner_params["login"], email: owner_params["mail"], password: user_password})
if interactor.success?
gitea_user = interactor.result
result = Gitea::User::GenerateTokenService.new(owner_params["login"], user_password).call
new_user.gitea_token = result['sha1']
new_user.gitea_uid = gitea_user['id']
user_mail = owner_params["mail"]
unless user_mail.present?
user_mail = "#{owner_params["login"]}_example@example.com"
end
new_user = User.new(owner_params.merge(platform: platform,mail: user_mail))
if new_user.save!
if owner_extension_params.present?
owner_extension_params = owner_extension_params["user_extensions"] if old_version_source.include?(platform) #trustie上需要
@ -109,14 +98,10 @@ class SyncForgeJob < ApplicationJob
end
def random_password
[*('a'..'z'),*(0..9),*('A'..'Z')].shuffle[0..8].join + ['$','#','&','*','@','_'].shuffle[0..2].join
end
#同步项目
def sync_projects(new_user,old_user_id, targets,platform)
def sync_projects(new_user,user_old_id, targets,platform)
Rails.logger.info("#######__projects_sync__start__########")
keys_to_delete = %w(id created_on updated_on user_id language)
keys_to_delete = %w(id created_on updated_on user_id language homepage_show image_url image_status audit_status_cd)
score_to_delete = %w(id created_at updated_at project_id)
begin
if targets.present?
@ -127,53 +112,44 @@ class SyncForgeJob < ApplicationJob
version_params = targets[:versions_params]
member_params = targets[:member_params]
project_score = targets[:project_score_params]
repo_params = targets[:repo_params]
praise_trends_params = targets[:praise_trends_params]
watchers_params = targets[:watchers_params]
if project.present?
project = project["project"] if old_version_source.include?(platform)
project_identifier = project["identifier"]
if repo_params.present?
project_identifier = repo_params["identifier"]
end
new_project = Project.new(project&.except!(*keys_to_delete).merge(user_id: new_user.id))
if new_project.save!(:validate => false)
if Project.exists?(identifier: project_identifier)
failed_dic = "public/sync_failed_users.dic"
File.open(failed_dic,"a") do |file|
file.puts "[\nTime---#{Time.now}\nproject_info---#{project}\nerrors---exists the same project:#{project_identifier}]\n "
if project_score.present?
project_score = project_score["project_score"] if old_version_source.include?(platform) #trustie上需要
ProjectScore.create!(project_score&.except!(*score_to_delete).merge(project_id: new_project.id))
end
else
new_project = Project.new(project&.except!(*keys_to_delete).merge(user_id: new_user.id))
if new_project.save!(:validate => false)
if project_identifier.present?
repository_params = {
hidden: project["is_public"],
user_id: new_user.id,
identifier: project_identifier
}
Repositories::CreateService.new(new_user, new_project, repository_params).call
end
if project_score.present?
project_score = project_score["project_score"] if old_version_source.include?(platform) #trustie上需要
ProjectScore.create!(project_score&.except!(*score_to_delete).merge(project_id: new_project.id))
end
sync_user_issues(new_project.id, issue_params, platform)
sync_members(new_project.id, member_params,platform)
sync_commits(new_project.id,new_project.gpid, commit_params,platform)
sync_pull_requests(new_project.id,new_user.id, pr_params,platform)
sync_versions(new_project.id, new_user.id, version_params,platform)
sync_watchers(new_project.id, watchers_params, platform)
sync_praises(new_project.id,praise_trends_params,platform)
sync_user_issues(new_project.id, issue_params, platform)
sync_members(new_project.id, member_params,platform)
sync_commits(new_project.id,new_project.gpid, commit_params,platform)
sync_pull_requests(new_project.id,new_user.id, pr_params,platform)
sync_versions(new_project.id, new_user.id, version_params,platform)
sync_watchers(new_project.id, watchers_params, platform)
sync_praises(new_project.id,praise_trends_params,platform)
end
Issue.select(:id, :assigned_to_id).where(id:@change_issue_ids, assigned_to_id: user_old_id)&.update_all(assigned_to_id: new_user.id)
Issue.select(:id, :author_id).where(id:@change_issue_ids, author_id: user_old_id)&.update_all(author_id: new_user.id)
Journal.select(:id,:user_id).where(id: @change_journal_ids, user_id: user_old_id)&.update_all(user_id: new_user.id)
Watcher.select(:id,:user_id).where(id: @change_watcher_ids, user_id: user_old_id)&.update_all(user_id: new_user.id)
PraiseTread.select(:id,:user_id).where(id: @change_praise_trend_ids, user_id: user_old_id)&.update_all(user_id: new_user.id)
PullRequest.select(:id,:user_id).where(id: @change_pr_ids, user_id: user_old_id)&.update_all(user_id: new_user.id)
Version.select(:id,:user_id).where(id: @change_version_ids, user_id: user_old_id)&.update_all(user_id: new_user.id)
end
end
end
rescue Exception => e
failed_dic = "public/sync_failed_users.dic"
File.open(failed_dic,"a") do |file|
file.puts "[\nTime---#{Time.now}\nproject_create_exception---\nerrors---project_created_failed:#{e}]\n "
end
Rails.logger.info("#######_______projects_sync__failed__#########{e}")
raise ActiveRecord::Rollback
end
end
@ -191,6 +167,7 @@ class SyncForgeJob < ApplicationJob
r = r["watcher"] if old_version_source.include?(platform)
new_wathcer = Watcher.new(r&.except!(*roles_other_delete).merge(watchable_id: project_id))
new_wathcer.save(:validate => false)
@change_watcher_ids.push(new_wathcer.id)
end
end
end
@ -217,6 +194,7 @@ class SyncForgeJob < ApplicationJob
r = r["praise_tread"] if old_version_source.include?(platform) #trustie上需要
new_tread = PraiseTread.new(r&.except!(*roles_other_delete).merge(praise_tread_object_id: project_id))
new_tread.save(:validate => false)
@change_praise_trend_ids.push(new_tread.id)
end
end
end
@ -230,34 +208,6 @@ class SyncForgeJob < ApplicationJob
end
def sync_roles(roles,platform)
Rails.logger.info("#######______sync_roles_start__#######")
roles_other_delete = %w(id)
ActiveRecord::Base.transaction do
begin
if roles.present?
Role.transaction do
roles.each do |r|
if r.present?
r = r["role"] if old_version_source.include?(platform) #trustie上需要
unless Role.exists?(name: r["name"])
Role.create!(r&.except!(*roles_other_delete))
end
end
end
end
end
Rails.logger.info("#######______sync_roles_end__#######")
rescue Exception => e
Rails.logger.info("#######_______sync_roles__failed__#########{e}")
raise ActiveRecord::Rollback
end
end
end
def sync_members(project_id,members_params,platform)
Rails.logger.info("#######______sync_members_start__#######")
member_to_delete = %w(id created_on project_id)
@ -331,6 +281,7 @@ class SyncForgeJob < ApplicationJob
issue_params = issue_params["issue"] if old_version_source.include?(platform) #trustie上需要
issue = Issue.new(issue_params&.except!(*issue_to_delete).merge(project_id: project_id))
if issue.save!(:validate => false)
@change_issue_ids.push(issue.id)
sync_journals(issue.id, jours_params, platform)
sync_commit_issues(issue.id,project_id, commit_params, platform)
else
@ -363,7 +314,7 @@ class SyncForgeJob < ApplicationJob
new_journal = new_journal["journal"] if old_version_source.include?(platform) #trustie上需要
new_journal = Journal.new(new_journal&.except!(*jour_to_delete).merge(journalized_id: issue_id))
if new_journal.save(:validate => false)
@change_journal_ids.push(new_journal.id)
if new_journal_detail.present?
sync_journal_details(new_journal_detail, new_journal.id, platform)
end
@ -445,8 +396,8 @@ class SyncForgeJob < ApplicationJob
i = i["pull_request"] if old_version_source.include?(platform) #trustie上需要
newpr = PullRequest.new(i&.except!(*commit_to_delete).merge(project_id: project_id))
newpr.save(:validate => false)
@change_pr_ids.push(newpr.id)
end
end
end
end
@ -497,6 +448,7 @@ class SyncForgeJob < ApplicationJob
i = i["version"] if old_version_source.include?(platform) #trustie上需要
new_v = Version.new(i&.except!(*version_to_delete).merge(project_id: project_id))
new_v.save!(:validate => false)
@change_version_ids.push(new_v.id)
end
end
end

3
app/models/user.rb
View File

@ -41,7 +41,8 @@ class User < ApplicationRecord
# educoder: 来自Educoder平台
# trustie: 来自Trustie平台
# forge: 平台本身注册的用户
enumerize :platform, in: [:forge, :educoder, :trustie], default: :forge, scope: :shallow
# military: 军科的用户
enumerize :platform, in: [:forge, :educoder, :trustie, :military], default: :forge, scope: :shallow
belongs_to :laboratory, optional: true

64
lib/tasks/generate_gitea_data.rake Normal file
View File

@ -0,0 +1,64 @@
desc "Generate gitea data"
namespace :create_gitea_data do
task created_user: :environment do
puts "__________begin_to_create_user_gitea_data___________"
gitea_users = User.where(platform: "military", gitea_uid: nil)
gitea_users.each_with_index do |u,index|
puts "___#{index+1}_______u.login:#{u.login}___________"
user_password = random_password
user_mail = u.mail ? u.mail : "#{u.login}_#{u.id}_example@example.com"
begin
interactor = Gitea::RegisterInteractor.call({username: u.login, email: user_mail, password: user_password})
if interactor.success?
gitea_user = interactor.result
result = Gitea::User::GenerateTokenService.new(u.login, user_password).call
u.gitea_token = result['sha1']
u.gitea_uid = gitea_user['id']
u.save!
puts "____creat_success______u.login:#{u.login}___________"
end
rescue Exception => e
failed_dic = "#{Rails.root}/public/sync_failed_users.dic"
File.open(failed_dic,"a") do |file|
file.puts "[\nTime---#{Time.now}\nuser_info---#{u.login}\nerrors--#{e}]\n "
end
puts "____creat_failed______u.login:#{u.login}_____message:#{e}______"
end
end
end
task created_repo: :environment do
puts "__________begin_to_create_repository_gitea_data___________"
user_ids = User.select(:id,:platform).all.where(platform: "military").pluck(:id)
gitea_projects = Project.where(user_id: user_ids)
gitea_projects.each_with_index do |p,index|
puts "___#{index+1}_______project.id:#{p.id}___________"
unless p.repository.present?
identifier = p.identifier ? p.identifier : "#{Time.now.to_i}_#{p.id}"
begin
repository_params = {
hidden: p["is_public"],
user_id: p.user_id,
identifier: identifier
}
Repositories::CreateService.new(p.owner, p, repository_params).call
puts "____creat_success______project.id:#{p.id}___________"
rescue Exception => e
failed_dic = "#{Rails.root}/public/sync_failed_users.dic"
File.open(failed_dic,"a") do |file|
file.puts "[\nTime---#{Time.now}\nproject_id---#{p.id}\nerrors--#{e}]\n "
end
end
end
end
end
def random_password
[*('a'..'z'),*(0..9),*('A'..'Z')].shuffle[0..8].join + ['$','#','&','*','@','_'].shuffle[0..2].join
end
end

2
lib/tasks/sync_table_structure.rake
View File

@ -4,7 +4,7 @@ namespace :sync_table_structure do
task import_csv: :environment do
puts "init table structure......."
system "mysql -uroot -p123456 -h127.0.0.1 forge_development < #{Rails.root}/db/structure.sql"
system "mysql -uroot -poracle10g -h127.0.0.1 forge_development < #{Rails.root}/db/structure.sql"
puts "init success"
end