From ad79fc38e6ac6e1a4f19cf563164d4f0462f6684 Mon Sep 17 00:00:00 2001 From: Kuzcoh Date: Thu, 7 Jul 2016 23:06:53 -0700 Subject: [PATCH 1/8] Added RealRepoStorageUsage.rb --- RealRepoStorageUsage/RealRepoStorageUsage.rb | 43 ++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 RealRepoStorageUsage/RealRepoStorageUsage.rb diff --git a/RealRepoStorageUsage/RealRepoStorageUsage.rb b/RealRepoStorageUsage/RealRepoStorageUsage.rb new file mode 100644 index 0000000..921958f --- /dev/null +++ b/RealRepoStorageUsage/RealRepoStorageUsage.rb @@ -0,0 +1,43 @@ +require "rubygems" +require "rest_client" +require "json" +require "filesize" + +if ARGV.empty? + puts "Usage: ruby RealUsage.rb $repositoryname" + exit +end + +repo = ARGV[0] #first argument should be repository to parse +user_name = 'admin' #change this +password= 'password' #change this +artifactory = "http://127.0.0.1:8081/artifactory/api/storage/" #change this +listfolders = "?list&deep=1&listFolders=1" + +url = artifactory + repo + listfolders +site = RestClient::Resource.new(url, user_name, password) +response = site.get(:accept=>"application/json") +string = response.body +parsed = JSON.parse(string) +parsed.map +array = Array.new +$i = 0 +artifacts = parsed["files"].count +totalbytes = 0 + +while $i < artifacts do + sha1 = parsed["files"][$i]["sha1"] + size = parsed["files"][$i]["size"] + array.push("#{size}:#{sha1}") + #grab sha1 and size of artifact, push to array + $i +=1 +end +array = array.uniq +#remove duplicates from array +array.each {|i| + re = /(.*):(.*)/ + m = re.match("#{i}") + totalbytes += m[1].to_i + #add up the total byte usage + } +puts "Total storage usage of repository " + repo + ": " + Filesize.from("#{totalbytes}" + "B").pretty \ No newline at end of file From cc3fe61ada2305c74dc7bd2699f3c99151594deb Mon Sep 17 00:00:00 2001 From: Omar Faizyar Date: Wed, 13 Jul 2016 15:22:19 -0700 Subject: [PATCH 2/8] Added Restore TrashCan Script Added Restore TrashCan Script --- REST-API-Examples/RestoreTrashCan.rb | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 REST-API-Examples/RestoreTrashCan.rb diff --git a/REST-API-Examples/RestoreTrashCan.rb b/REST-API-Examples/RestoreTrashCan.rb new file mode 100644 index 0000000..cbea160 --- /dev/null +++ b/REST-API-Examples/RestoreTrashCan.rb @@ -0,0 +1,28 @@ +require "rubygems" +require "rest_client" +require "json" +#This script restores every file inside the Trashcan +user_name = 'admin' #change this +password= 'password' #change this +repo = "auto-trashcan" +artifactory = "http://127.0.0.1:8081/artifactory/api/storage/" #change this +listfolders = "?list&deep=1&listFolders=1" +url = artifactory + repo + listfolders +site = RestClient::Resource.new(url, user_name, password) +response = site.get(:accept=>"application/json") +string = response.body +parsed = JSON.parse(string) +parsed.map + + +while true + i = 0 + trashedartifacts = parsed["files"][i]["uri"] + restoreurl = "http://127.0.0.1:8081/artifactory/api/trash/restore" + trashedartifacts + "?to=" + trashedartifacts #change my artifactory home + site2 = RestClient::Resource.new(restoreurl, user_name, password) + response2 = site2.post(:accept=>"application/json") + string2 = response2.body + puts string2 + i+=1 +end + From df6f32f40669a8bb5f83063f3c0e98302b69c9c0 Mon Sep 17 00:00:00 2001 From: Omar Faizyar Date: Tue, 19 Jul 2016 09:57:03 -0700 Subject: [PATCH 3/8] Fixed artifactory.py Fixed artifactory.py --- ArtifactoryLogParser/artifactory.py | 43 ++++++++++------------------- 1 file changed, 14 insertions(+), 29 deletions(-) diff --git a/ArtifactoryLogParser/artifactory.py b/ArtifactoryLogParser/artifactory.py index 6b8ff27..f85a7f9 100644 --- a/ArtifactoryLogParser/artifactory.py +++ b/ArtifactoryLogParser/artifactory.py @@ -9,42 +9,27 @@ sys.exit(0) + readfile = sys.argv[1] i = 0 with open(readfile) as f: for line in f: - if "WARN" in line: - try: - regex = '^([-0-9]+ [:0-9]+,[0-9]+) \[([-a-zA-Z0-9]+)\] \[([A-Z]+) *\] \(([.a-zA-Z0-9]+):([0-9]+)\) - (.*)$' - match = re.search(regex,line.encode('ascii'),flags = 0) - print Fore.RED + "WARNING:" + (match.group(6)) - - except Exception: - pass - -with open(readfile) as f: - for line in f: - if "ERROR" in line: - try: - regex = '^([-0-9]+ [:0-9]+,[0-9]+) \[([-a-zA-Z0-9]+)\] \[([A-Z]+) *\] \(([.a-zA-Z0-9]+):([0-9]+)\) - (.*)$' - match = re.search(regex,line.encode('ascii'),flags = 0) - print Fore.YELLOW + "ERROR:" + (match.group(6)) - - except Exception: - pass - -with open(readfile) as f: - for line in f: - if "INFO" in line: - try: + try: + if "WARN" in line: + regex = '^([-0-9]+ [:0-9]+,[0-9]+) \[([-a-zA-Z0-9]+)\] \[([A-Z]+) *\] \(([.a-zA-Z0-9]+):([0-9]+)\) - (.*)$' + match = re.search(regex,line.encode('ascii'),flags = 0) + print Fore.RED + "WARNING:" + (match.group(6)) + elif "ERROR" in line: + regex = '^([-0-9]+ [:0-9]+,[0-9]+) \[([-a-zA-Z0-9]+)\] \[([A-Z]+) *\] \(([.a-zA-Z0-9]+):([0-9]+)\) - (.*)$' + match = re.search(regex,line.encode('ascii'),flags = 0) + print Fore.YELLOW + "ERROR:" + (match.group(6)) + elif "INFO" in line: regex = '^([-0-9]+ [:0-9]+,[0-9]+) \[([-a-zA-Z0-9]+)\] \[([A-Z]+) *\] \(([.a-zA-Z0-9]+):([0-9]+)\) - (.*)$' match = re.search(regex,line.encode('ascii'),flags = 0) print Fore.GREEN + "INFO:" + (match.group(6)) - - except Exception: - pass - + except Exception: + pass print (Style.RESET_ALL) exit() - + \ No newline at end of file From 76a7c7f8d33875a2368ced9a3dededa98f9897b9 Mon Sep 17 00:00:00 2001 From: Kuzcoh Date: Mon, 14 Nov 2016 22:23:08 -0800 Subject: [PATCH 4/8] added dictionary and sorting --- ArtifactoryLogParser/artifactory.py | 58 ++++++++++++++++++----------- ArtifactoryLogParser/request.py | 6 +++ 2 files changed, 42 insertions(+), 22 deletions(-) diff --git a/ArtifactoryLogParser/artifactory.py b/ArtifactoryLogParser/artifactory.py index f85a7f9..faf9b13 100644 --- a/ArtifactoryLogParser/artifactory.py +++ b/ArtifactoryLogParser/artifactory.py @@ -3,33 +3,47 @@ import sys import colorama from colorama import Fore, Back, Style +import operator if len(sys.argv) !=2: print('Usage: python artifactory.py artifactory.log') sys.exit(0) +errors = [] +warnings = [] +readfile = sys.argv[1] +def readlogs(): + with open(readfile) as f: + for line in f: + try: + if "WARN" in line: + regex = '^([-0-9]+ [:0-9]+,[0-9]+) \[([-a-zA-Z0-9]+)\] \[([A-Z]+) *\] \(([.a-zA-Z0-9]+):([0-9]+)\) - (.*)$' + match = re.search(regex,line.encode('ascii'),flags = 0) + print Fore.RED + (match.group(6)) + warnings.append(match.group(6)[:40]) -readfile = sys.argv[1] -i = 0 -with open(readfile) as f: - for line in f: - try: - if "WARN" in line: - regex = '^([-0-9]+ [:0-9]+,[0-9]+) \[([-a-zA-Z0-9]+)\] \[([A-Z]+) *\] \(([.a-zA-Z0-9]+):([0-9]+)\) - (.*)$' - match = re.search(regex,line.encode('ascii'),flags = 0) - print Fore.RED + "WARNING:" + (match.group(6)) - elif "ERROR" in line: - regex = '^([-0-9]+ [:0-9]+,[0-9]+) \[([-a-zA-Z0-9]+)\] \[([A-Z]+) *\] \(([.a-zA-Z0-9]+):([0-9]+)\) - (.*)$' - match = re.search(regex,line.encode('ascii'),flags = 0) - print Fore.YELLOW + "ERROR:" + (match.group(6)) - elif "INFO" in line: - regex = '^([-0-9]+ [:0-9]+,[0-9]+) \[([-a-zA-Z0-9]+)\] \[([A-Z]+) *\] \(([.a-zA-Z0-9]+):([0-9]+)\) - (.*)$' - match = re.search(regex,line.encode('ascii'),flags = 0) - print Fore.GREEN + "INFO:" + (match.group(6)) - except Exception: - pass -print (Style.RESET_ALL) + elif "ERROR" in line: + regex = '^([-0-9]+ [:0-9]+,[0-9]+) \[([-a-zA-Z0-9]+)\] \[([A-Z]+) *\] \(([.a-zA-Z0-9]+):([0-9]+)\) - (.*)$' + match = re.search(regex,line.encode('ascii'),flags = 0) + print Fore.YELLOW + (match.group(6)) + errors.append(match.group(6)[:40]) + elif "INFO" in line: + regex = '^([-0-9]+ [:0-9]+,[0-9]+) \[([-a-zA-Z0-9]+)\] \[([A-Z]+) *\] \(([.a-zA-Z0-9]+):([0-9]+)\) - (.*)$' + match = re.search(regex,line.encode('ascii'),flags = 0) + print Fore.GREEN + (match.group(6)) + except Exception: + pass + warningdic = dict((x,warnings.count(x)) for x in set(warnings)) + + for key, value in sorted(warningdic.iteritems(), key=lambda (k,v): (v,k)): + print Fore.RED + "%s : times --- %s" % (key, value) + + errorsdic = dict((x,errors.count(x)) for x in set(errors)) + for key, value in sorted(errorsdic.iteritems(), key=lambda (k,v): (v,k)): + print Fore.YELLOW + "%s : times --- %s" % (key, value) + print (Style.RESET_ALL) + +readlogs() exit() - - \ No newline at end of file + diff --git a/ArtifactoryLogParser/request.py b/ArtifactoryLogParser/request.py index 28a3cc7..e76c252 100644 --- a/ArtifactoryLogParser/request.py +++ b/ArtifactoryLogParser/request.py @@ -1,14 +1,20 @@ import sys import re +import colorama +from colorama import Fore, Back, Style #match.group(1) = time(example: 20160621065454) #match.group(4) = IP Address #match.group(6) = HTTP Method(GET,POST,PUT) #match.group(7) = URL #match.group(9) = HTTP Response Code +responsecodes = [] + + if len(sys.argv) !=2: print('Usage: python request.py request.log') sys.exit(0) + readfile = sys.argv[1] with open(readfile) as f: for line in f: From d30e20f2ddd0e2c289dcf616eed34d0d1466876e Mon Sep 17 00:00:00 2001 From: Kuzcoh Date: Mon, 14 Nov 2016 22:23:21 -0800 Subject: [PATCH 5/8] added dictionary and sorting --- ArtifactoryLogParser/request.py | 34 +++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/ArtifactoryLogParser/request.py b/ArtifactoryLogParser/request.py index e76c252..708b64b 100644 --- a/ArtifactoryLogParser/request.py +++ b/ArtifactoryLogParser/request.py @@ -16,12 +16,30 @@ sys.exit(0) readfile = sys.argv[1] -with open(readfile) as f: - for line in f: - try: - p = re.compile(ur'(\d*)\|(\d*)\|(.*)\|(.*)\|(.*)\|(.*)\|(.*)\|(.*)\|(.*)\|(.*)') - match = re.match(p,line) - print match.group(4) + " - " + match.group(6) + " - " + match.group(9) - except Exception: - pass + +def readlogs(): + with open(readfile) as f: + for line in f: + try: + p = re.compile(ur'(\d*)\|(\d*)\|(.*)\|(.*)\|(.*)\|(.*)\|(.*)\|(.*)\|(.*)\|(.*)') + match = re.match(p,line) + response = match.group(4) + " - " + match.group(6) + " - " + match.group(9) + if match.group(9)[:1] == "2": + print Fore.GREEN + response + responsecodes.append(match.group(9)) + elif match.group(9)[:1] == "4" or "5": + print Fore.RED + response + responsecodes.append(match.group(9)) + else: + print response + responsecodes.append(match.group(9)) + except Exception: + pass + print (Style.RESET_ALL) + responsedic = dict((x,responsecodes.count(x)) for x in set(responsecodes)) + for key, value in sorted(responsedic.iteritems(), key=lambda (k,v): (v,k)): + print "%s: %s" % (key, value) + + +readlogs() exit() \ No newline at end of file From 47f2f1334b8d71a37ca63105276db33279f76972 Mon Sep 17 00:00:00 2001 From: Omar Faiz Date: Tue, 15 Nov 2016 09:26:22 -0800 Subject: [PATCH 6/8] artifactory.py can parse debuggers now --- ArtifactoryLogParser/artifactory.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) mode change 100644 => 100755 ArtifactoryLogParser/artifactory.py diff --git a/ArtifactoryLogParser/artifactory.py b/ArtifactoryLogParser/artifactory.py old mode 100644 new mode 100755 index faf9b13..c788cbf --- a/ArtifactoryLogParser/artifactory.py +++ b/ArtifactoryLogParser/artifactory.py @@ -11,6 +11,7 @@ errors = [] warnings = [] +debug = [] readfile = sys.argv[1] def readlogs(): @@ -32,16 +33,26 @@ def readlogs(): regex = '^([-0-9]+ [:0-9]+,[0-9]+) \[([-a-zA-Z0-9]+)\] \[([A-Z]+) *\] \(([.a-zA-Z0-9]+):([0-9]+)\) - (.*)$' match = re.search(regex,line.encode('ascii'),flags = 0) print Fore.GREEN + (match.group(6)) + elif "DEBUG" in line: + regex = '^([-0-9]+ [:0-9]+,[0-9]+) \[([-a-zA-Z0-9]+)\] \[([A-Z]+) *\] \(([.a-zA-Z0-9]+):([0-9]+)\) - (.*)$' + match = re.search(regex,line.encode('ascii'),flags = 0) + print Fore.MAGENTA + line + debug.append(match.group(6)[:40]) except Exception: pass warningdic = dict((x,warnings.count(x)) for x in set(warnings)) - for key, value in sorted(warningdic.iteritems(), key=lambda (k,v): (v,k)): print Fore.RED + "%s : times --- %s" % (key, value) errorsdic = dict((x,errors.count(x)) for x in set(errors)) for key, value in sorted(errorsdic.iteritems(), key=lambda (k,v): (v,k)): print Fore.YELLOW + "%s : times --- %s" % (key, value) + + debugdic = dict((x,debug.count(x)) for x in set(debug)) + for key,value in sorted(debugdic.iteritems(),key=lambda (k,v):(v,k)): + print Fore.MAGENTA + "%s : times --- %s" % (key,value) + + print (Style.RESET_ALL) readlogs() From 31dab774e05a3758def1ea73649d1f5d66c7a768 Mon Sep 17 00:00:00 2001 From: kuzcoh Date: Tue, 14 Feb 2017 14:10:12 -0800 Subject: [PATCH 7/8] Added buildcleaner By customer request --- BuildCleaner/README.md | 3 +++ BuildCleaner/buildcleaner.py | 44 ++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 BuildCleaner/README.md create mode 100644 BuildCleaner/buildcleaner.py diff --git a/BuildCleaner/README.md b/BuildCleaner/README.md new file mode 100644 index 0000000..d96f93e --- /dev/null +++ b/BuildCleaner/README.md @@ -0,0 +1,3 @@ +# Build Cleaner +Deletes any builds without a set status such as 'Staged' and 'Release' +Artifactory URL and credentials need to be set first, please use an 'Admin' user \ No newline at end of file diff --git a/BuildCleaner/buildcleaner.py b/BuildCleaner/buildcleaner.py new file mode 100644 index 0000000..9da6dbe --- /dev/null +++ b/BuildCleaner/buildcleaner.py @@ -0,0 +1,44 @@ +import requests +import json +import sys +import re + +username = "admin" +password = "password" +artifactory = "https://127.0.0.1:8081/artifactory/" # change this artifactory URL +#make sure to clarify if HTTP or HTTPS + +def deletebuild(buildtodelete,idtodelete): + print buildtodelete + idtodelete + r = requests.delete(buildtodelete + idtodelete, auth= (username,password)) + +def inspectbuildjson(buildjson,buildjsonid): + #looks through every single build json in every build ID + r = requests.get(buildjson+buildjsonid, auth = (username, password)) + buildinfo = json.loads(r.text) + + if "statuses" not in buildinfo["buildInfo"] or len(buildinfo["buildInfo"]["statuses"]) == 0: + print "Deleting this build: " + buildjson + buildjsonid + deletebuild(buildjson,re.sub('/', '?buildNumbers=', buildjsonid)) + else: + print "Keeping this build: " + buildjson + buildjsonid + " - " + buildinfo["buildInfo"]["statuses"][0]["status"] + +def findbuildnumbers(buildname): + #finds all build ID's from all build projects + url = buildname + r = requests.get(url, auth = (username, password)) + build_data = json.loads(r.text) + for item in build_data["buildsNumbers"]: + inspectbuildjson(url,item["uri"]) + +def getallbuilds(): + #looks through all build projects + buildurl = "api/build" + url = artifactory + buildurl + r = requests.get(url, auth = (username, password)) + json_data = json.loads(r.text) + for item in json_data["builds"]: + findbuildnumbers(url + item["uri"]) + + +getallbuilds() \ No newline at end of file From 10141fb59c9b89ac8512a2bcb09ca6b9d606ada2 Mon Sep 17 00:00:00 2001 From: kuzcoh Date: Tue, 14 Feb 2017 14:15:25 -0800 Subject: [PATCH 8/8] Removed ruby script --- RealRepoStorageUsage/RealRepoStorageUsage.rb | 43 -------------------- 1 file changed, 43 deletions(-) delete mode 100644 RealRepoStorageUsage/RealRepoStorageUsage.rb diff --git a/RealRepoStorageUsage/RealRepoStorageUsage.rb b/RealRepoStorageUsage/RealRepoStorageUsage.rb deleted file mode 100644 index 921958f..0000000 --- a/RealRepoStorageUsage/RealRepoStorageUsage.rb +++ /dev/null @@ -1,43 +0,0 @@ -require "rubygems" -require "rest_client" -require "json" -require "filesize" - -if ARGV.empty? - puts "Usage: ruby RealUsage.rb $repositoryname" - exit -end - -repo = ARGV[0] #first argument should be repository to parse -user_name = 'admin' #change this -password= 'password' #change this -artifactory = "http://127.0.0.1:8081/artifactory/api/storage/" #change this -listfolders = "?list&deep=1&listFolders=1" - -url = artifactory + repo + listfolders -site = RestClient::Resource.new(url, user_name, password) -response = site.get(:accept=>"application/json") -string = response.body -parsed = JSON.parse(string) -parsed.map -array = Array.new -$i = 0 -artifacts = parsed["files"].count -totalbytes = 0 - -while $i < artifacts do - sha1 = parsed["files"][$i]["sha1"] - size = parsed["files"][$i]["size"] - array.push("#{size}:#{sha1}") - #grab sha1 and size of artifact, push to array - $i +=1 -end -array = array.uniq -#remove duplicates from array -array.each {|i| - re = /(.*):(.*)/ - m = re.match("#{i}") - totalbytes += m[1].to_i - #add up the total byte usage - } -puts "Total storage usage of repository " + repo + ": " + Filesize.from("#{totalbytes}" + "B").pretty \ No newline at end of file