From 931f2ba0be3c8bbb6d97c5a5fc0de66c747bce99 Mon Sep 17 00:00:00 2001 From: Alexey Lustin Date: Sun, 15 May 2016 17:08:39 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9F=D0=B5=D1=80=D0=B2=D1=8B=D0=B9=20=D0=BF?= =?UTF-8?q?=D0=BE=D0=B4=D1=85=D0=BE=D0=B4=20=D0=BA=20#1=20=D0=BF=D0=BE=20?= =?UTF-8?q?=D1=81=D0=B5=D1=80=D1=8C=D0=B5=D0=B7=D0=BD=D0=BE=D0=BC=D1=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 - data/README.md | 73 +++++++++++++++++---- tools/github-search.rb | 143 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 204 insertions(+), 13 deletions(-) create mode 100644 tools/github-search.rb diff --git a/.gitignore b/.gitignore index a562a47..e69de29 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +0,0 @@ -tools/*.rb diff --git a/data/README.md b/data/README.md index d290e2e..08c45fb 100644 --- a/data/README.md +++ b/data/README.md @@ -1,11 +1,11 @@ # BSL и OScript репозиторий и их пользователи -15/05/2016 16:34 +15/05/2016 17:04 ## Пользователи -Общее количество пользователей: 10 +Общее количество пользователей: 29 | Пользователь | Акаунт | |---|---| @@ -15,14 +15,33 @@ |StanislavDementev| https://github.com/StanislavDementev | |carlosthebest| https://github.com/carlosthebest | |mishatre| https://github.com/mishatre | -|EvilBeaver| https://github.com/EvilBeaver | +|stop-time| https://github.com/stop-time | +|vpozdnyakov| https://github.com/vpozdnyakov | +|xmentosx| https://github.com/xmentosx | +|po4ta56| https://github.com/po4ta56 | +|nixel2007| https://github.com/nixel2007 | +|Petr28| https://github.com/Petr28 | +|SINAPTeam| https://github.com/SINAPTeam | +|1C-Company| https://github.com/1C-Company | +|DmZhDNK| https://github.com/DmZhDNK | +|Nikiforov001| https://github.com/Nikiforov001 | +|SmokyAce| https://github.com/SmokyAce | +|1cto| https://github.com/1cto | +|skyksandr| https://github.com/skyksandr | +|silverbulleters| https://github.com/silverbulleters | +|1c-expert| https://github.com/1c-expert | +|poleznieglup0sti| https://github.com/poleznieglup0sti | +|a-sitnikov| https://github.com/a-sitnikov | +|wizi4d| https://github.com/wizi4d | +|SergeFocus| https://github.com/SergeFocus | +|ilgizvip| https://github.com/ilgizvip | +|temsan| https://github.com/temsan | |xDrivenDevelopment| https://github.com/xDrivenDevelopment | -|nextkmv| https://github.com/nextkmv | -|otdel| https://github.com/otdel | +|michaelbag| https://github.com/michaelbag | ## Репозитории BSL (1C) -Общее количество репозиториев: 12 +Общее количество репозиториев: 40 | Репозиторий | URL | |---|---| @@ -36,23 +55,53 @@ |carlosthebest/test_2| https://github.com/carlosthebest/test_2 | |carlosthebest/test| https://github.com/carlosthebest/test | |mishatre/UnitedDB| https://github.com/mishatre/UnitedDB | +|stop-time/pm_ut11| https://github.com/stop-time/pm_ut11 | +|vpozdnyakov/DesktopServiceDesk| https://github.com/vpozdnyakov/DesktopServiceDesk | +|xmentosx/test2| https://github.com/xmentosx/test2 | +|po4ta56/conf_mkp| https://github.com/po4ta56/conf_mkp | +|nixel2007/test_epf| https://github.com/nixel2007/test_epf | +|nixel2007/atom-language-1c-bsl| https://github.com/nixel2007/atom-language-1c-bsl | +|Petr28/EDT_Test| https://github.com/Petr28/EDT_Test | +|SINAPTeam/sinap| https://github.com/SINAPTeam/sinap | +|1C-Company/dt-demo-configuration| https://github.com/1C-Company/dt-demo-configuration | +|DmZhDNK/Home_Bookkeeping| https://github.com/DmZhDNK/Home_Bookkeeping | +|Nikiforov001/EDT_1.0| https://github.com/Nikiforov001/EDT_1.0 | +|SmokyAce/integration-1c-redmine| https://github.com/SmokyAce/integration-1c-redmine | +|1cto/Test_8.3.7| https://github.com/1cto/Test_8.3.7 | +|skyksandr/ext-netlenka| https://github.com/skyksandr/ext-netlenka | +|silverbulleters/vanessa-dbaqa| https://github.com/silverbulleters/vanessa-dbaqa | +|silverbulleters/vanessa-stack-commons| https://github.com/silverbulleters/vanessa-stack-commons | +|silverbulleters/vanessa-behavior| https://github.com/silverbulleters/vanessa-behavior | +|silverbulleters/vanessa-runner| https://github.com/silverbulleters/vanessa-runner | +|1c-expert/TrainingManagement| https://github.com/1c-expert/TrainingManagement | +|poleznieglup0sti/perfect-nutrition-1C| https://github.com/poleznieglup0sti/perfect-nutrition-1C | +|a-sitnikov/erp2_xtests| https://github.com/a-sitnikov/erp2_xtests | +|wizi4d/TaskManagerFor1C| https://github.com/wizi4d/TaskManagerFor1C | +|SergeFocus/VedomostAmortizNMA| https://github.com/SergeFocus/VedomostAmortizNMA | +|SergeFocus/1C-Functin-to-yEd| https://github.com/SergeFocus/1C-Functin-to-yEd | +|ilgizvip/git-city| https://github.com/ilgizvip/git-city | +|temsan/ATIDocsClient.BP30| https://github.com/temsan/ATIDocsClient.BP30 | +|temsan/ATIDocsClient.BP20| https://github.com/temsan/ATIDocsClient.BP20 | |xDrivenDevelopment/xUnitFor1C| https://github.com/xDrivenDevelopment/xUnitFor1C | |xDrivenDevelopment/atom-language-1c-bsl| https://github.com/xDrivenDevelopment/atom-language-1c-bsl | +|michaelbag/1ctest| https://github.com/michaelbag/1ctest | ## Репозитории OS (1Script) -Общее количество репозиториев: 10 +Общее количество репозиториев: 12 | Репозиторий | URL | |---|---| -|EvilBeaver/OneScript| https://github.com/EvilBeaver/OneScript | -|EvilBeaver/oscript-library| https://github.com/EvilBeaver/oscript-library | -|EvilBeaver/oscript-app-template| https://github.com/EvilBeaver/oscript-app-template | +|nixel2007/gitlab-ci-for-1c| https://github.com/nixel2007/gitlab-ci-for-1c | +|nixel2007/1c-syntax| https://github.com/nixel2007/1c-syntax | +|1cto/Test_8.3.7| https://github.com/1cto/Test_8.3.7 | +|silverbulleters/vanessa-unpack| https://github.com/silverbulleters/vanessa-unpack | +|silverbulleters/oscript-epf-merge| https://github.com/silverbulleters/oscript-epf-merge | +|silverbulleters/vanessa-runner| https://github.com/silverbulleters/vanessa-runner | +|silverbulleters/vanessa-behavior| https://github.com/silverbulleters/vanessa-behavior | |xDrivenDevelopment/1script| https://github.com/xDrivenDevelopment/1script | |xDrivenDevelopment/precommit1c| https://github.com/xDrivenDevelopment/precommit1c | |xDrivenDevelopment/v83unpack| https://github.com/xDrivenDevelopment/v83unpack | |xDrivenDevelopment/AutoAdmin1C| https://github.com/xDrivenDevelopment/AutoAdmin1C | |xDrivenDevelopment/1c-syntax| https://github.com/xDrivenDevelopment/1c-syntax | -|nextkmv/vServer| https://github.com/nextkmv/vServer | -|otdel/OneScript| https://github.com/otdel/OneScript | diff --git a/tools/github-search.rb b/tools/github-search.rb new file mode 100644 index 0000000..c603fef --- /dev/null +++ b/tools/github-search.rb @@ -0,0 +1,143 @@ +require 'nokogiri' +require 'open-uri' +require 'openssl' + +require 'octokit' +require 'faraday-http-cache' + +users = [] + +page = 0 +has_user = true + +stack = Faraday::RackBuilder.new do |builder| + builder.use Faraday::HttpCache + builder.use Octokit::Response::RaiseError + builder.adapter Faraday.default_adapter +end + +["bsl", "os"].each do |extension_file| + +user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.854.0 Safari/535.2" + +while has_user do + has_user = false + page = page + 1 + + p "Search the " + extension_file + " on page " + page.to_s + + #https://github.com/search?p=1&q=%D0%9F%D1%80%D0%BE%D1%86%D0%B5%D0%B4%D1%83%D1%80%D0%B0+in%3Afile+extension%3Absl&ref=searchresults&type=Code&utf8=%E2%9C%93 + #https://github.com/search?utf8=✓&q=Процедура+in%3Afile+extension%3Absl + url = "https://github.com/search?p=" + page.to_s+ "&q=%D0%9F%D1%80%D0%BE%D1%86%D0%B5%D0%B4%D1%83%D1%80%D0%B0+in%3Afile+extension%3A" + extension_file + "&ref=searchresults&type=Code&utf8=%E2%9C%93" + p url + + selectors = [ + '#code_search_results > div.code-list > div' + ] + + begin + doc = Nokogiri::HTML(open(url,'User-Agent' => user_agent,ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE)) + rescue + p "End of page or GitHub ban the search request" + break + end + + selectors.each do |selector| + doc.css(selector).each do |div| + div.children.each do |child| + case child.name + when 'a' + has_user = true + user = child["href"] + user.slice! "/" + + p "find the user " + user + + users.push(user) + end + end + end + end + + sleep 10 + +end +end + +Octokit.middleware = stack +Octokit.auto_paginate = true + +Octokit.configure do |c| + c.login = ENV['GITHUB_USER'] + c.password = ENV['GITHUB_PASSWORD'] + c.connection_options[:ssl] = { :verify => false } +end + +repos_os = [] +repos_bsl = [] + +users.uniq.each do |user| + + p "Proces repos for " + user + + #Процедура user:allustin in:file extension:bsl + bls_result = Octokit.search_code("Процедура user:" + user + " in:file extension:bsl") + + bls_result["items"].each do |res| + repos_bsl.push(res["repository"]["full_name"]) + end + + os_result = Octokit.search_code("Процедура user:" + user + " in:file extension:os") + + os_result["items"].each do |res| + repos_os.push(res["repository"]["full_name"]) + end + + sleep 13 + +end + +date_time = DateTime.now.strftime("%d/%m/%Y %H:%M\n") + +user_on_markdown = File.expand_path(File.join(File.dirname(__FILE__), "../data/README.md")) + +File.open(user_on_markdown,"w") do |f| + f << "# BSL и OScript репозиторий и их пользователи\n" + f << "\n" + f << "" + date_time + "\n" + f << "\n" + f << "## Пользователи\n" + f << "\n" + f << "Общее количество пользователей: " + users.uniq.size.to_s + "\n" + f << "\n" + f << "| Пользователь | Акаунт |\n" + f << "|---|---|\n" + users.uniq.each do |user| + f << "|" + user + "| https://github.com/" + user + " |\n" + end + f << "\n" + f << "## Репозитории BSL (1C)\n" + f << "\n" + f << "Общее количество репозиториев: " + repos_bsl.uniq.size.to_s + "\n" + f << "\n" + f << "| Репозиторий | URL |\n" + f << "|---|---|\n" + repos_bsl.uniq.each do |repo| + f << "|" + repo + "| https://github.com/" + repo + " |\n" + end + f << "\n" + f << "## Репозитории OS (1Script)\n" + f << "\n" + f << "Общее количество репозиториев: " + repos_os.uniq.size.to_s + "\n" + f << "\n" + f << "| Репозиторий | URL |\n" + f << "|---|---|\n" + repos_os.uniq.each do |repo| + f << "|" + repo + "| https://github.com/" + repo + " |\n" + end + f << "\n" +end + + + +