Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(50)

Side by Side Diff: tools/auto_bisect/bisect_perf_regression.py

Issue 1625573004: Update legacy bisect to post results to perf dashboard. (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@fix_import_path
Patch Set: update doc Created 4 years, 10 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright 2013 The Chromium Authors. All rights reserved. 2 # Copyright 2013 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """Chromium auto-bisect tool 6 """Chromium auto-bisect tool
7 7
8 This script bisects a range of commits using binary search. It starts by getting 8 This script bisects a range of commits using binary search. It starts by getting
9 reference values for the specified "good" and "bad" commits. Then, for revisions 9 reference values for the specified "good" and "bad" commits. Then, for revisions
10 in between, it will get builds, run tests and classify intermediate revisions as 10 in between, it will get builds, run tests and classify intermediate revisions as
(...skipping 23 matching lines...) Expand all
34 --good_revision 408222\ 34 --good_revision 408222\
35 --bad_revision 408232\ 35 --bad_revision 408232\
36 --bisect_mode return_code\ 36 --bisect_mode return_code\
37 --builder_type full 37 --builder_type full
38 38
39 In practice, the auto-bisect tool is usually run on tryserver.chromium.perf 39 In practice, the auto-bisect tool is usually run on tryserver.chromium.perf
40 try bots, and is started by tools/run-bisect-perf-regression.py using 40 try bots, and is started by tools/run-bisect-perf-regression.py using
41 config parameters from tools/auto_bisect/bisect.cfg. 41 config parameters from tools/auto_bisect/bisect.cfg.
42 """ 42 """
43 43
44 import argparse
44 import copy 45 import copy
45 import errno 46 import errno
46 import hashlib 47 import hashlib
48 import json
47 import logging 49 import logging
48 import argparse
49 import os 50 import os
50 import re 51 import re
51 import shlex 52 import shlex
52 import shutil 53 import shutil
53 import StringIO 54 import StringIO
54 import sys 55 import sys
55 import time 56 import time
57 import urllib2
56 58
57 sys.path.append(os.path.join( 59 sys.path.append(os.path.join(
58 os.path.dirname(__file__), os.path.pardir, 'telemetry')) 60 os.path.dirname(__file__), os.path.pardir, 'telemetry'))
59 61
60 from bisect_printer import BisectPrinter 62 from bisect_printer import BisectPrinter
61 from bisect_results import BisectResults 63 from bisect_results import BisectResults
64 import bisect_results_json
62 from bisect_state import BisectState 65 from bisect_state import BisectState
63 import bisect_utils 66 import bisect_utils
64 import builder 67 import builder
65 import fetch_build 68 import fetch_build
66 import math_utils 69 import math_utils
67 import query_crbug 70 import query_crbug
68 import request_build 71 import request_build
69 import source_control 72 import source_control
70 73
71 # The script is in chromium/src/tools/auto_bisect. Throughout this script, 74 # The script is in chromium/src/tools/auto_bisect. Throughout this script,
(...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after
117 BISECT_TRYJOB_BRANCH = 'bisect-tryjob' 120 BISECT_TRYJOB_BRANCH = 'bisect-tryjob'
118 # Git master branch name. 121 # Git master branch name.
119 BISECT_MASTER_BRANCH = 'master' 122 BISECT_MASTER_BRANCH = 'master'
120 # File to store 'git diff' content. 123 # File to store 'git diff' content.
121 BISECT_PATCH_FILE = 'deps_patch.txt' 124 BISECT_PATCH_FILE = 'deps_patch.txt'
122 # SVN repo where the bisect try jobs are submitted. 125 # SVN repo where the bisect try jobs are submitted.
123 PERF_SVN_REPO_URL = 'svn://svn.chromium.org/chrome-try/try-perf' 126 PERF_SVN_REPO_URL = 'svn://svn.chromium.org/chrome-try/try-perf'
124 FULL_SVN_REPO_URL = 'svn://svn.chromium.org/chrome-try/try' 127 FULL_SVN_REPO_URL = 'svn://svn.chromium.org/chrome-try/try'
125 ANDROID_CHROME_SVN_REPO_URL = ('svn://svn.chromium.org/chrome-try-internal/' 128 ANDROID_CHROME_SVN_REPO_URL = ('svn://svn.chromium.org/chrome-try-internal/'
126 'try-perf') 129 'try-perf')
130 PERF_DASH_RESULTS_URL = 'https://chromeperf.appspot.com/post_bisect_results'
127 131
128 132
129 class RunGitError(Exception): 133 class RunGitError(Exception):
130 134
131 def __str__(self): 135 def __str__(self):
132 return '%s\nError executing git command.' % self.args[0] 136 return '%s\nError executing git command.' % self.args[0]
133 137
134 138
135 def GetSHA1HexDigest(contents): 139 def GetSHA1HexDigest(contents):
136 """Returns SHA1 hex digest of the given string.""" 140 """Returns SHA1 hex digest of the given string."""
(...skipping 2361 matching lines...) Expand 10 before | Expand all | Expand 10 after
2498 opts: The options parsed from the command line. 2502 opts: The options parsed from the command line.
2499 2503
2500 Returns: 2504 Returns:
2501 True if the platform and build system are supported. 2505 True if the platform and build system are supported.
2502 """ 2506 """
2503 # Haven't tested the script out on any other platforms yet. 2507 # Haven't tested the script out on any other platforms yet.
2504 supported = ['posix', 'nt'] 2508 supported = ['posix', 'nt']
2505 return os.name in supported 2509 return os.name in supported
2506 2510
2507 2511
2512 def _PostBisectResults(bisect_results, opts, src_cwd):
2513 """Posts bisect results to Perf Dashboard."""
2514 bisect_utils.OutputAnnotationStepStart('Post Results')
2515
2516 results = bisect_results_json.get(
2517 bisect_results, opts, DepotDirectoryRegistry(src_cwd))
2518 data = {'data': results}
2519 request = urllib2.Request(PERF_DASH_RESULTS_URL)
2520 request.add_header('Content-Type', 'application/json')
2521 try:
2522 urllib2.urlopen(request, json.dumps(data))
2523 except urllib2.URLError as e:
2524 print 'Failed to post bisect results. Error: %s.' % e
2525 bisect_utils.OutputAnnotationStepWarning()
2526
2527 bisect_utils.OutputAnnotationStepClosed()
2528
2529
2508 def RemoveBuildFiles(build_type): 2530 def RemoveBuildFiles(build_type):
2509 """Removes build files from previous runs.""" 2531 """Removes build files from previous runs."""
2510 out_dir = os.path.join('out', build_type) 2532 out_dir = os.path.join('out', build_type)
2511 build_dir = os.path.join('build', build_type) 2533 build_dir = os.path.join('build', build_type)
2512 logging.info('Removing build files in "%s" and "%s".', 2534 logging.info('Removing build files in "%s" and "%s".',
2513 os.path.abspath(out_dir), os.path.abspath(build_dir)) 2535 os.path.abspath(out_dir), os.path.abspath(build_dir))
2514 try: 2536 try:
2515 RemakeDirectoryTree(out_dir) 2537 RemakeDirectoryTree(out_dir)
2516 RemakeDirectoryTree(build_dir) 2538 RemakeDirectoryTree(build_dir)
2517 except Exception as e: 2539 except Exception as e:
(...skipping 59 matching lines...) Expand 10 before | Expand all | Expand 10 after
2577 self.debug_ignore_perf_test = None 2599 self.debug_ignore_perf_test = None
2578 self.debug_ignore_regression_confidence = None 2600 self.debug_ignore_regression_confidence = None
2579 self.debug_fake_first_test_mean = 0 2601 self.debug_fake_first_test_mean = 0
2580 self.target_arch = 'ia32' 2602 self.target_arch = 'ia32'
2581 self.target_build_type = 'Release' 2603 self.target_build_type = 'Release'
2582 self.builder_type = 'perf' 2604 self.builder_type = 'perf'
2583 self.bisect_mode = bisect_utils.BISECT_MODE_MEAN 2605 self.bisect_mode = bisect_utils.BISECT_MODE_MEAN
2584 self.improvement_direction = 0 2606 self.improvement_direction = 0
2585 self.bug_id = '' 2607 self.bug_id = ''
2586 self.required_initial_confidence = 80.0 2608 self.required_initial_confidence = 80.0
2609 self.try_job_id = None
2587 2610
2588 @staticmethod 2611 @staticmethod
2589 def _AddBisectOptionsGroup(parser): 2612 def _AddBisectOptionsGroup(parser):
2590 group = parser.add_argument_group('Bisect options') 2613 group = parser.add_argument_group('Bisect options')
2591 group.add_argument('-c', '--command', required=True, 2614 group.add_argument('-c', '--command', required=True,
2592 help='A command to execute your performance test at ' 2615 help='A command to execute your performance test at '
2593 'each point in the bisection.') 2616 'each point in the bisection.')
2594 group.add_argument('-b', '--bad_revision', required=True, 2617 group.add_argument('-b', '--bad_revision', required=True,
2595 help='A bad revision to start bisection. Must be later ' 2618 help='A bad revision to start bisection. Must be later '
2596 'than good revision. May be either a git or svn ' 2619 'than good revision. May be either a git or svn '
(...skipping 254 matching lines...) Expand 10 before | Expand all | Expand 10 after
2851 not opts.debug_ignore_sync and 2874 not opts.debug_ignore_sync and
2852 not opts.working_directory): 2875 not opts.working_directory):
2853 raise RuntimeError('You must switch to master branch to run bisection.') 2876 raise RuntimeError('You must switch to master branch to run bisection.')
2854 bisect_test = BisectPerformanceMetrics(opts, os.getcwd()) 2877 bisect_test = BisectPerformanceMetrics(opts, os.getcwd())
2855 try: 2878 try:
2856 results = bisect_test.Run(opts.command, opts.bad_revision, 2879 results = bisect_test.Run(opts.command, opts.bad_revision,
2857 opts.good_revision, opts.metric) 2880 opts.good_revision, opts.metric)
2858 if results.error: 2881 if results.error:
2859 raise RuntimeError(results.error) 2882 raise RuntimeError(results.error)
2860 bisect_test.printer.FormatAndPrintResults(results) 2883 bisect_test.printer.FormatAndPrintResults(results)
2884 _PostBisectResults(results, opts, os.getcwd())
2861 return 0 2885 return 0
2862 finally: 2886 finally:
2863 bisect_test.PerformCleanup() 2887 bisect_test.PerformCleanup()
2864 except RuntimeError as e: 2888 except RuntimeError as e:
2865 if opts.output_buildbot_annotations: 2889 if opts.output_buildbot_annotations:
2866 # The perf dashboard scrapes the "results" step in order to comment on 2890 # The perf dashboard scrapes the "results" step in order to comment on
2867 # bugs. If you change this, please update the perf dashboard as well. 2891 # bugs. If you change this, please update the perf dashboard as well.
2868 bisect_utils.OutputAnnotationStepStart('Results') 2892 bisect_utils.OutputAnnotationStepStart('Results')
2869 print 'Runtime Error: %s' % e 2893 print 'Runtime Error: %s' % e
2870 if opts.output_buildbot_annotations: 2894 if opts.output_buildbot_annotations:
2871 bisect_utils.OutputAnnotationStepClosed() 2895 bisect_utils.OutputAnnotationStepClosed()
2872 return 1 2896 return 1
2873 2897
2874 2898
2875 if __name__ == '__main__': 2899 if __name__ == '__main__':
2876 sys.exit(main()) 2900 sys.exit(main())
OLDNEW
« no previous file with comments | « no previous file | tools/auto_bisect/bisect_perf_regression_test.py » ('j') | tools/auto_bisect/bisect_perf_regression_test.py » ('J')

Powered by Google App Engine
This is Rietveld 408576698