Revision 1572444361982199fdab9c6f6b7e94383717b6c9 authored by Johannes Schindelin on 15 January 2019, 18:23:03 UTC, committed by Johannes Schindelin on 29 January 2019, 11:05:59 UTC
The fact that Git's test suite is implemented in Unix shell script that is as portable as we can muster, combined with the fact that Unix shell scripting is foreign to Windows (and therefore has to be emulated), results in pretty abysmal speed of the test suite on that platform, for pretty much no other reason than that language choice. For comparison: while the Linux build & test is typically done within about 8 minutes, the Windows build & test typically lasts about 80 minutes in Azure Pipelines. To help with that, let's use the Azure Pipeline feature where you can parallelize jobs, make jobs depend on each other, and pass artifacts between them. The tests are distributed using the following heuristic: listing all test scripts ordered by size in descending order (as a cheap way to estimate the overall run time), every Nth script is run (where N is the total number of parallel jobs), starting at the index corresponding to the parallel job. This slicing is performed by a new function that is added to the `test-tool`. To optimize the overall runtime of the entire Pipeline, we need to move the Windows jobs to the beginning (otherwise there would be a very decent chance for the Pipeline to be run only the Windows build, while all the parallel Windows test jobs wait for this single one). We use Azure Pipelines Artifacts for both the minimal Git for Windows SDK as well as the built executables, as deduplication and caching close to the agents makes that really fast. For comparison: while downloading and unpacking the minimal Git for Windows SDK via PowerShell takes only one minute (down from anywhere between 2.5 to 7 when using a shallow clone), uploading it as Pipeline Artifact takes less than 30s and downloading and unpacking less than 20s (sometimes even as little as only twelve seconds). Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>
1 parent 3532811
gpg-interface.c
#include "cache.h"
#include "config.h"
#include "run-command.h"
#include "strbuf.h"
#include "gpg-interface.h"
#include "sigchain.h"
#include "tempfile.h"
static char *configured_signing_key;
struct gpg_format {
const char *name;
const char *program;
const char **verify_args;
const char **sigs;
};
static const char *openpgp_verify_args[] = {
"--keyid-format=long",
NULL
};
static const char *openpgp_sigs[] = {
"-----BEGIN PGP SIGNATURE-----",
"-----BEGIN PGP MESSAGE-----",
NULL
};
static const char *x509_verify_args[] = {
NULL
};
static const char *x509_sigs[] = {
"-----BEGIN SIGNED MESSAGE-----",
NULL
};
static struct gpg_format gpg_format[] = {
{ .name = "openpgp", .program = "gpg",
.verify_args = openpgp_verify_args,
.sigs = openpgp_sigs
},
{ .name = "x509", .program = "gpgsm",
.verify_args = x509_verify_args,
.sigs = x509_sigs
},
};
static struct gpg_format *use_format = &gpg_format[0];
static struct gpg_format *get_format_by_name(const char *str)
{
int i;
for (i = 0; i < ARRAY_SIZE(gpg_format); i++)
if (!strcmp(gpg_format[i].name, str))
return gpg_format + i;
return NULL;
}
static struct gpg_format *get_format_by_sig(const char *sig)
{
int i, j;
for (i = 0; i < ARRAY_SIZE(gpg_format); i++)
for (j = 0; gpg_format[i].sigs[j]; j++)
if (starts_with(sig, gpg_format[i].sigs[j]))
return gpg_format + i;
return NULL;
}
void signature_check_clear(struct signature_check *sigc)
{
FREE_AND_NULL(sigc->payload);
FREE_AND_NULL(sigc->gpg_output);
FREE_AND_NULL(sigc->gpg_status);
FREE_AND_NULL(sigc->signer);
FREE_AND_NULL(sigc->key);
FREE_AND_NULL(sigc->fingerprint);
FREE_AND_NULL(sigc->primary_key_fingerprint);
}
/* An exclusive status -- only one of them can appear in output */
#define GPG_STATUS_EXCLUSIVE (1<<0)
/* The status includes key identifier */
#define GPG_STATUS_KEYID (1<<1)
/* The status includes user identifier */
#define GPG_STATUS_UID (1<<2)
/* The status includes key fingerprints */
#define GPG_STATUS_FINGERPRINT (1<<3)
/* Short-hand for standard exclusive *SIG status with keyid & UID */
#define GPG_STATUS_STDSIG (GPG_STATUS_EXCLUSIVE|GPG_STATUS_KEYID|GPG_STATUS_UID)
static struct {
char result;
const char *check;
unsigned int flags;
} sigcheck_gpg_status[] = {
{ 'G', "GOODSIG ", GPG_STATUS_STDSIG },
{ 'B', "BADSIG ", GPG_STATUS_STDSIG },
{ 'U', "TRUST_NEVER", 0 },
{ 'U', "TRUST_UNDEFINED", 0 },
{ 'E', "ERRSIG ", GPG_STATUS_EXCLUSIVE|GPG_STATUS_KEYID },
{ 'X', "EXPSIG ", GPG_STATUS_STDSIG },
{ 'Y', "EXPKEYSIG ", GPG_STATUS_STDSIG },
{ 'R', "REVKEYSIG ", GPG_STATUS_STDSIG },
{ 0, "VALIDSIG ", GPG_STATUS_FINGERPRINT },
};
static void parse_gpg_output(struct signature_check *sigc)
{
const char *buf = sigc->gpg_status;
const char *line, *next;
int i, j;
int seen_exclusive_status = 0;
/* Iterate over all lines */
for (line = buf; *line; line = strchrnul(line+1, '\n')) {
while (*line == '\n')
line++;
/* Skip lines that don't start with GNUPG status */
if (!skip_prefix(line, "[GNUPG:] ", &line))
continue;
/* Iterate over all search strings */
for (i = 0; i < ARRAY_SIZE(sigcheck_gpg_status); i++) {
if (skip_prefix(line, sigcheck_gpg_status[i].check, &line)) {
if (sigcheck_gpg_status[i].flags & GPG_STATUS_EXCLUSIVE) {
if (seen_exclusive_status++)
goto found_duplicate_status;
}
if (sigcheck_gpg_status[i].result)
sigc->result = sigcheck_gpg_status[i].result;
/* Do we have key information? */
if (sigcheck_gpg_status[i].flags & GPG_STATUS_KEYID) {
next = strchrnul(line, ' ');
free(sigc->key);
sigc->key = xmemdupz(line, next - line);
/* Do we have signer information? */
if (*next && (sigcheck_gpg_status[i].flags & GPG_STATUS_UID)) {
line = next + 1;
next = strchrnul(line, '\n');
free(sigc->signer);
sigc->signer = xmemdupz(line, next - line);
}
}
/* Do we have fingerprint? */
if (sigcheck_gpg_status[i].flags & GPG_STATUS_FINGERPRINT) {
next = strchrnul(line, ' ');
free(sigc->fingerprint);
sigc->fingerprint = xmemdupz(line, next - line);
/* Skip interim fields */
for (j = 9; j > 0; j--) {
if (!*next)
break;
line = next + 1;
next = strchrnul(line, ' ');
}
next = strchrnul(line, '\n');
free(sigc->primary_key_fingerprint);
sigc->primary_key_fingerprint = xmemdupz(line, next - line);
}
break;
}
}
}
return;
found_duplicate_status:
/*
* GOODSIG, BADSIG etc. can occur only once for each signature.
* Therefore, if we had more than one then we're dealing with multiple
* signatures. We don't support them currently, and they're rather
* hard to create, so something is likely fishy and we should reject
* them altogether.
*/
sigc->result = 'E';
/* Clear partial data to avoid confusion */
FREE_AND_NULL(sigc->primary_key_fingerprint);
FREE_AND_NULL(sigc->fingerprint);
FREE_AND_NULL(sigc->signer);
FREE_AND_NULL(sigc->key);
}
int check_signature(const char *payload, size_t plen, const char *signature,
size_t slen, struct signature_check *sigc)
{
struct strbuf gpg_output = STRBUF_INIT;
struct strbuf gpg_status = STRBUF_INIT;
int status;
sigc->result = 'N';
status = verify_signed_buffer(payload, plen, signature, slen,
&gpg_output, &gpg_status);
if (status && !gpg_output.len)
goto out;
sigc->payload = xmemdupz(payload, plen);
sigc->gpg_output = strbuf_detach(&gpg_output, NULL);
sigc->gpg_status = strbuf_detach(&gpg_status, NULL);
parse_gpg_output(sigc);
status |= sigc->result != 'G' && sigc->result != 'U';
out:
strbuf_release(&gpg_status);
strbuf_release(&gpg_output);
return !!status;
}
void print_signature_buffer(const struct signature_check *sigc, unsigned flags)
{
const char *output = flags & GPG_VERIFY_RAW ?
sigc->gpg_status : sigc->gpg_output;
if (flags & GPG_VERIFY_VERBOSE && sigc->payload)
fputs(sigc->payload, stdout);
if (output)
fputs(output, stderr);
}
size_t parse_signature(const char *buf, size_t size)
{
size_t len = 0;
size_t match = size;
while (len < size) {
const char *eol;
if (get_format_by_sig(buf + len))
match = len;
eol = memchr(buf + len, '\n', size - len);
len += eol ? eol - (buf + len) + 1 : size - len;
}
return match;
}
void set_signing_key(const char *key)
{
free(configured_signing_key);
configured_signing_key = xstrdup(key);
}
int git_gpg_config(const char *var, const char *value, void *cb)
{
struct gpg_format *fmt = NULL;
char *fmtname = NULL;
if (!strcmp(var, "user.signingkey")) {
if (!value)
return config_error_nonbool(var);
set_signing_key(value);
return 0;
}
if (!strcmp(var, "gpg.format")) {
if (!value)
return config_error_nonbool(var);
fmt = get_format_by_name(value);
if (!fmt)
return error("unsupported value for %s: %s",
var, value);
use_format = fmt;
return 0;
}
if (!strcmp(var, "gpg.program") || !strcmp(var, "gpg.openpgp.program"))
fmtname = "openpgp";
if (!strcmp(var, "gpg.x509.program"))
fmtname = "x509";
if (fmtname) {
fmt = get_format_by_name(fmtname);
return git_config_string(&fmt->program, var, value);
}
return 0;
}
const char *get_signing_key(void)
{
if (configured_signing_key)
return configured_signing_key;
return git_committer_info(IDENT_STRICT|IDENT_NO_DATE);
}
int sign_buffer(struct strbuf *buffer, struct strbuf *signature, const char *signing_key)
{
struct child_process gpg = CHILD_PROCESS_INIT;
int ret;
size_t i, j, bottom;
struct strbuf gpg_status = STRBUF_INIT;
argv_array_pushl(&gpg.args,
use_format->program,
"--status-fd=2",
"-bsau", signing_key,
NULL);
bottom = signature->len;
/*
* When the username signingkey is bad, program could be terminated
* because gpg exits without reading and then write gets SIGPIPE.
*/
sigchain_push(SIGPIPE, SIG_IGN);
ret = pipe_command(&gpg, buffer->buf, buffer->len,
signature, 1024, &gpg_status, 0);
sigchain_pop(SIGPIPE);
ret |= !strstr(gpg_status.buf, "\n[GNUPG:] SIG_CREATED ");
strbuf_release(&gpg_status);
if (ret)
return error(_("gpg failed to sign the data"));
/* Strip CR from the line endings, in case we are on Windows. */
for (i = j = bottom; i < signature->len; i++)
if (signature->buf[i] != '\r') {
if (i != j)
signature->buf[j] = signature->buf[i];
j++;
}
strbuf_setlen(signature, j);
return 0;
}
int verify_signed_buffer(const char *payload, size_t payload_size,
const char *signature, size_t signature_size,
struct strbuf *gpg_output, struct strbuf *gpg_status)
{
struct child_process gpg = CHILD_PROCESS_INIT;
struct gpg_format *fmt;
struct tempfile *temp;
int ret;
struct strbuf buf = STRBUF_INIT;
temp = mks_tempfile_t(".git_vtag_tmpXXXXXX");
if (!temp)
return error_errno(_("could not create temporary file"));
if (write_in_full(temp->fd, signature, signature_size) < 0 ||
close_tempfile_gently(temp) < 0) {
error_errno(_("failed writing detached signature to '%s'"),
temp->filename.buf);
delete_tempfile(&temp);
return -1;
}
fmt = get_format_by_sig(signature);
if (!fmt)
BUG("bad signature '%s'", signature);
argv_array_push(&gpg.args, fmt->program);
argv_array_pushv(&gpg.args, fmt->verify_args);
argv_array_pushl(&gpg.args,
"--status-fd=1",
"--verify", temp->filename.buf, "-",
NULL);
if (!gpg_status)
gpg_status = &buf;
sigchain_push(SIGPIPE, SIG_IGN);
ret = pipe_command(&gpg, payload, payload_size,
gpg_status, 0, gpg_output, 0);
sigchain_pop(SIGPIPE);
delete_tempfile(&temp);
ret |= !strstr(gpg_status->buf, "\n[GNUPG:] GOODSIG ");
strbuf_release(&buf); /* no matter it was used or not */
return ret;
}
Computing file changes ...