https://github.com/CryptDB/cryptdb
Tip revision: 7678bc98d3054f1418371779c6d1050cd1a88b2e authored by Raluca Ada Popa on 04 January 2014, 01:31:06 UTC
small changes to readme
small changes to readme
Tip revision: 7678bc9
test.cc
#include <algorithm>
#include <stdio.h>
#include <unistd.h>
#include <time.h>
#include <string.h>
#include <iostream>
#include <istream>
#include <fstream>
#include <iomanip>
#include <pthread.h>
#include <signal.h>
#include <stdlib.h>
#include <sys/time.h>
#include <time.h>
#include <sys/wait.h>
#include <main/Connect.hh>
#include <util/util.hh>
#include <util/params.hh>
#include <util/cryptdb_log.hh>
#include <test/test_utils.hh>
#include <test/TestQueries.hh>
using namespace NTL;
/*static string __attribute__((unused))
padPasswd(const string &s)
{
string r = s;
r.resize(AES_KEY_BYTES, '0');
return r;
}
static clock_t timeStart;
static void
startTimer ()
{
timeStart = time(NULL);
}
static double
readTimer()
{
clock_t currentTime = time(NULL);
double res = (double) (currentTime - timeStart) * 1000.0;
return res;
}
static void __attribute__((unused))
test_OPE()
{
cerr << "\n OPE test started \n";
const unsigned int OPEPlaintextSize = 32;
const unsigned int OPECiphertextSize = 128;
unsigned char key[AES_KEY_SIZE/
bitsPerByte] =
{158, 242, 169, 240, 255, 166, 39, 177, 149, 166, 190, 237, 178, 254, 187,
40};
cerr <<"Key is " << stringToByteInts(string((char *) key,
AES_KEY_SIZE/bitsPerByte)) << "\n";
OPE * ope = new OPE((const char *) key, OPEPlaintextSize,
OPECiphertextSize);
unsigned char plaintext[OPEPlaintextSize/bitsPerByte] = {74, 95, 221, 84};
string plaintext_s = string((char *) plaintext,
OPEPlaintextSize/bitsPerByte);
string ciphertext = ope->encrypt(plaintext_s);
string decryption = ope->decrypt(ciphertext);
LOG(test) << "Plaintext is " << stringToByteInts(plaintext_s);
LOG(test) << "Ciphertext is " << stringToByteInts(ciphertext);
LOG(test) << "Decryption is " << stringToByteInts(decryption);
myassert(plaintext_s == decryption, "OPE test failed \n");
cerr << "OPE Test Succeeded \n";
unsigned int tests = 100;
cerr << "Started " << tests << " tests \n ";
clock_t encTime = 0;
clock_t decTime = 0;
clock_t currTime;
time_t startTime = time(NULL);
for (unsigned int i = 0; i < tests; i++) {
string ptext = randomBytes(OPEPlaintextSize/bitsPerByte);
currTime = clock();
string ctext = ope->encrypt(ptext);
encTime += clock() - currTime;
currTime = clock();
ope->decrypt(ctext);
decTime += clock() - currTime;
}
time_t endTime = time(NULL);
cout << "(time): encrypt/decrypt take " <<
(1.0 * (double) (endTime-startTime))/(2.0*tests) << "s \n";
cout << "encrypt takes on average " <<
((double) encTime*1000.0)/(1.0*CLOCKS_PER_SEC*tests) << "ms \n";
cout << "decrypt takes on average " <<
((double) decTime*1000.0)/(1.0*CLOCKS_PER_SEC*tests) << "ms \n";
}
static void __attribute__((unused))
evaluate_AES(const TestConfig &tc, int argc, char ** argv)
{
if (argc!=2) {
cout << "usage ./test noTests \n";
exit(1);
}
unsigned int notests = 10;
string key = randomBytes(AES_KEY_SIZE/bitsPerByte);
string ptext = randomBytes(AES_BLOCK_SIZE/bitsPerByte);
AES_KEY aesKey;
AES_set_encrypt_key((const uint8_t *) key.c_str(), AES_KEY_SIZE, &aesKey);
timeval startTime, endTime;
unsigned int tests = 1024*1024;
for (unsigned int j = 0; j < notests; j++) {
gettimeofday(&startTime, NULL);
for (unsigned int i = 0; i < tests; i++) {
unsigned char ctext[AES_BLOCK_SIZE/bitsPerByte];
AES_encrypt((const uint8_t *) ptext.c_str(), ctext, &aesKey);
ptext = string((char *) ctext, AES_BLOCK_BYTES);
}
gettimeofday(&endTime, NULL);
cerr << (tests*16.0)/(1024*1024) << " "
<< timeInSec(startTime, endTime) << "\n";
tests = (uint) ((double) tests * 1.2);
}
cout << "result " << ptext << "\n";
}
static void __attribute__((unused))
test_HGD()
{
unsigned int len = 16; //bytes
unsigned int bitsPrecision = len * bitsPerByte + 10;
ZZ K = ZZFromStringFast(padForZZ(randomBytes(len)));
ZZ N1 = ZZFromStringFast(padForZZ(randomBytes(len)));
ZZ N2 = ZZFromStringFast(padForZZ(randomBytes(len)));
ZZ SEED = ZZFromStringFast(padForZZ(randomBytes(len)));
ZZ sample = HGD(K, N1, N2, SEED, len*bitsPerByte, bitsPrecision);
LOG(test) << "N1 is " << stringToByteInts(StringFromZZ(N1));
LOG(test) << "N2 is " << stringToByteInts(StringFromZZ(N2));
LOG(test) << "K is " << stringToByteInts(StringFromZZ(K));
LOG(test) << "HGD sample is " << stringToByteInts(StringFromZZ(sample));
unsigned int tests = 1000;
cerr << " Started " << tests << " tests \n";
clock_t totalTime = 0; //in clock ticks
for (unsigned int i = 0; i< tests; i++) {
K = N1+ N2+1;
while (K > N1+N2) {
cerr << "test " << i << "\n";
K = ZZFromStringFast(padForZZ(randomBytes(len)));
N1 = ZZFromStringFast(padForZZ(randomBytes(len)));
N2 = ZZFromStringFast(padForZZ(randomBytes(len)));
SEED = ZZFromStringFast(padForZZ(randomBytes(len)));
}
clock_t currentTime = clock();
sample = HGD(K, N1, N2, SEED, len*bitsPerByte, bitsPrecision);
totalTime += clock() - currentTime;
}
cerr << "average milliseconds per test is " <<
((double) totalTime * 1000.0) / ((double) tests * CLOCKS_PER_SEC) << "\n";
}*/
/*
void test_EDBProxy_noSecurity() {
EDBProxy e = EDBProxy((char *)"dbname = postgres", false);
PGresult * res = e.execute("SELECT * FROM pg_database ;");
int nFields = PQnfields(res);
for (int i = 0; i < nFields; i++)
printf("%-15s", PQfname(res, i));
printf("\n\n");
*/
/* next, print out the rows */
/* for (int i = 0; i < PQntuples(res); i++)
{
for (int j = 0; j < nFields; j++)
printf("%-15s", PQgetvalue(res, i, j));
printf("\n");
}
PQclear(res);
}*/
/*
static void __attribute__((unused))
evaluateMetrics(const TestConfig &tc, int argc, char ** argv)
{
if (argc != 4) {
printf("usage: ./test noRecords tests haveindex?(0/1) ");
exit(1);
}
unsigned int noRecords = atoi(argv[1]);
unsigned int tests = atoi(argv[2]);
time_t timerStart, timerEnd;
EDBProxy * cl = new EDBProxy(tc.host, tc.user, tc.pass, tc.db);
cl->execute(
"CREATE TABLE testplain (field1 int, field2 int, field3 int);");
cl->execute(
"CREATE TABLE testcipher (field1 varchar(16), field2 varchar(16), field3 varchar(16));");
timerStart = time(NULL);
//populate both tables with increasing values
for (unsigned int i = 0; i < noRecords; i++) {
string commandPlain = "INSERT INTO testplain VALUES (";
string value = StringFromVal(i);
commandPlain = commandPlain + value + ", " + value + "," + value +
");";
cl->execute(commandPlain);
}
timerEnd = time(NULL);
printf("insert plain average time %f ms \n",
(1000.0 * (double) (timerEnd-timerStart))/(noRecords*1.0));
timerStart = time(NULL);
for (unsigned int i = 0; i < noRecords; i++) {
string commandCipher = "INSERT INTO testcipher VALUES (";
string valueBytes ="'" +
StringFromVal(i, AES_BLOCK_BITS/
bitsPerByte) + "'";
commandCipher = commandCipher + valueBytes + ", " + valueBytes +
", " + valueBytes+ ");";
cl->execute(commandCipher);
}
timerEnd = time(NULL);
printf("insert cipher average time %f ms \n",
(1000.0 * (double) (timerEnd-timerStart))/(noRecords*1.0));
if (atoi(argv[3]) == 1) {
cout << "create index";
cl->execute("CREATE INDEX indplain ON testplain (field1) ;");
cl->execute("CREATE INDEX indcipher ON testcipher (field1) ;");
}
timerStart = time(NULL);
//equality selection
for (unsigned int i = 0; i < tests; i++) {
int j = rand() % noRecords;
string commandPlain = "SELECT * FROM testplain WHERE field1 = ";
string value = StringFromVal(j);
commandPlain += value + ";";
//cout << "CL " << clock() << "\n";
cl->execute(commandPlain);
}
timerEnd = time(NULL);
printf("select plain time %f ms \n",
(1000.0 * (double) (timerEnd-timerStart))/(tests*1.0));
timerStart = time(NULL);
//equality selection
for (unsigned int i = 0; i < tests; i++) {
int j = rand() % noRecords;
string commandCipher = "SELECT * FROM testcipher WHERE field1 = ";
string valueBytes = "'" +
StringFromVal(j, AES_BLOCK_BITS/
bitsPerByte) + "'";
commandCipher = commandCipher + valueBytes + ";";
cl->execute(commandCipher);
}
timerEnd = time(NULL);
printf("cipher average time %f ms \n",
(1000.0* (double) (timerEnd-timerStart))/(tests*1.0));
/ *
timerStart = time(NULL);
//inequality selection
for (int i = 0; i < tests; i++) {
int leftJ = rand() % noRecords;
//int rightJ = rand() % noRecords;
//if (leftJ > rightJ) {
// int aux = leftJ;
// leftJ = rightJ;
// rightJ = aux;
//}
int rightJ = leftJ + (rand() % 50);
string commandPlain = "SELECT * FROM testplain WHERE field1 > ";
string leftJBytes = StringFromVal(leftJ);
string rightJBytes = StringFromVal(rightJ);
commandPlain = commandPlain + leftJBytes + " AND field1 < " +
rightJBytes + ";";
cl->execute(commandPlain);
}
timerEnd = time(NULL);
printf("range select plain %f ms \n",
(1000.0*(timerEnd-timerStart))/(tests*1.0));
* /
/ *
timerStart = time(NULL);
//inequality selection
for (int i = 0; i < tests; i++) {
int leftJ = rand() % noRecords;
//int rightJ = rand() % noRecords;
//if (leftJ > rightJ) {
// int aux = leftJ;
// leftJ = rightJ;
// rightJ = aux;
//}
int rightJ = leftJ + (rand() % 50);
string commandCipher = "SELECT * FROM testcipher WHERE field1 > ";
string leftJBytes = "'" + StringFromVal(leftJ,
AES_BLOCK_BITS/bitsPerByte) + "'";
string rightJBytes = "'" + StringFromVal(rightJ,
AES_BLOCK_BITS/bitsPerByte) + "'";
commandCipher = commandCipher + leftJBytes + " AND field1 < " +
rightJBytes + ";";
cl->execute(commandCipher);
}
timerEnd = time(NULL);
printf("range select cipher %f ms \n",
(1000.0*(timerEnd-timerStart))/(tests*1.0));
* /
cl->execute("DROP TABLE testplain;");
cl->execute("DROP TABLE testcipher;");
}
*/
//tests protected methods of EDBProxy
/*class tester : public EDBProxy {
public:
tester(const TestConfig &tc, const string &masterKey) : EDBProxy(tc.host,
tc.user,
tc.pass,
tc.db,
0, false)
{
setMasterKey(masterKey);
}
tester(const TestConfig &tc) : EDBProxy(tc.host, tc.user, tc.pass, tc.db)
{
};
void testClientParser();
void loadData(EDBProxy * cl, string workload, int logFreq);
//void testMarshallBinary();
};
void
tester::testClientParser()
{
cerr << "testClientParser uses old EDBProxy -- not run" << endl;
list<string> queries = list<string>();
//queries.push_back(string("CREATE TABLE people (id integer, age integer,
// name integer);"));
queries.push_back(string(
"CREATE TABLE city (name integer, citizen integer);"));
queries.push_back(string(
"CREATE TABLE emp (id integer, name text, age integer, job text);"));
//queries.push_back(string("SELECT city.citizen FROM people, city WHERE
// city.citizen = people.name ; "));
//queries.push_back(string("INSERT INTO people VALUES (5, 23, 34);"));
//queries.push_back(string("INSERT INTO city VALUES (34, 24);"));
//queries.push_back(string("SELECT people.id FROM people WHERE people.id =
// 5 AND people.id = people.age ;"));
//queries.push_back(string("DROP TABLE people;"));
list<int> expectedCount = list<int>();
expectedCount.push_back(1);
expectedCount.push_back(1);
expectedCount.push_back(4);
expectedCount.push_back(2);
expectedCount.push_back(4);
expectedCount.push_back(1);
list<string> expected = list<string>();
expected.push_back("CREATE TABLE table0 ( field0DET integer, field0OPE bigint, field1DET integer, field1OPE bigint, field2DET integer, field2OPE bigint );");
expected.push_back("CREATE TABLE table1 ( field0DET integer, field0OPE bigint, field1DET integer, field1OPE bigint );");
expected.push_back("UPDATE table1 SET field1DET = DECRYPT(0);");
expected.push_back("UPDATE table0 SET field2DET = DECRYPT(0);");
expected.push_back("UPDATE table1 SET field1DET = EQUALIZE(0);");
expected.push_back("SELECT table1.field1DET FROM table0, table1 WHERE table1.field1DET = table0.field2DET ;");
expected.push_back("UPDATE table1 SET field1DET = 5;");
expected.push_back("UPDATE table1 SET field1OPE = 5;");
expected.push_back("UPDATE table0 SET field0DET = DECRYPT(0);");
expected.push_back("UPDATE table0 SET field1DET = DECRYPT(0);");
expected.push_back("UPDATE table0 SET field0DET = EQUALIZE(0);");
expected.push_back("SELECT table0.field0DET FROM table0 WHERE table0.field0DET = 5 AND table0.field0DET = table0.field1DET ;");
expected.push_back("DROP TABLE table0;");
list<string>::iterator it = queries.begin();
for (; it != queries.end(); it++) { //TODO: check against expected...at
// this point is more of a manual
// check
bool temp;
list<string> response = rewriteEncryptQuery(it->c_str(), temp);
LOG(test) << "query issued/response: " << *it << ", " << toString(response, stringToByteInts);
}
exit();
cerr << "TEST TRANSLATOR PASSED \n";
}
*/
/*
static void __attribute__((unused))
testCryptoManager()
{
string masterKey = randomBytes(AES_KEY_BYTES);
CryptoManager * cm = new CryptoManager(masterKey);
cerr << "TEST CRYPTO MANAGER \n";
//test marshall and unmarshall key
string m = cm->marshallKey(masterKey);
LOG(test) << "master key is " << stringToByteInts(masterKey);
LOG(test) << "marshall is " << m;
string masterKey2 = cm->unmarshallKey(m);
myassert(masterKey == masterKey2, "marshall test failed");
LOG(test) << "key for field1: "
<< stringToByteInts(cm->getKey("field1", SECLEVEL::OPE));
LOG(test) << "key for table5.field12OPE:"
<< stringToByteInts(cm->getKey("table5.field12OPE", SECLEVEL::OPE));
//test SEM
AES_KEY * aesKey = cm->get_key_SEM(masterKey);
uint64_t salt = 3953954;
uint32_t value = 5;
uint32_t eValue = cm->encrypt_SEM(value, aesKey, salt);
cerr << "\n sem encr of " << value << " is " << eValue << "with salt " <<
salt << " and decr of encr is " <<
cm->decrypt_SEM(eValue, aesKey, salt) <<"\n";
myassert(cm->decrypt_SEM(eValue, aesKey,
salt) == value,
"decrypt of encrypt does not return value");
cerr << "SEMANTIC " << (int) SECLEVEL::OPE << "\n";
uint64_t value2 = 10;
uint64_t eValue2 = cm->encrypt_SEM(value2, aesKey, salt);
cerr << "sem encr of " << value2 << " is " << eValue2 <<
" and signed is " << (int64_t) eValue2 << "\n";
myassert(cm->decrypt_SEM(eValue2, aesKey,
salt) == value2,
"decrypt of encrypt does not return correct value for uint64_t");
cerr << "0, " << (int64_t) eValue2 << ", " << m << ", " << salt << "\n";
OPE * ope = cm->get_key_OPE(masterKey);
uint64_t eOPE = cm->encrypt_OPE(value, ope);
cerr << "encryption is eOPE " << eOPE << " \n";
myassert(cm->decrypt_OPE(eOPE, ope) == value, "ope failed");
cerr << "TEST CRYPTO MANAGER PASSED \n";
}*/
static void
testTrain(const TestConfig &tc, int ac, char **a) {
/*EDBProxy * pr = new EDBProxy(tc.host, tc.user, tc.pass, tc.db, tc.port, false, true);
pr->plain_execute("drop database cryptdbtest;");
pr->plain_execute("create database cryptdbtest;");
pr->plain_execute("use cryptdbtest;");
string mkey = randomBytes(AES_BLOCK_BYTES);
pr->setMasterKey(mkey);
pr->execute("train 1 testtraincreate testpatterns 1");
LOG(test) << "\n \n ";
LOG(test) << "done training";
LOG(test) << "\n \n ";
ifstream fc("testtraincreate");
while (!fc.eof()) {
string query = getQuery(fc);
pr->plain_execute(query);
}
fc.close();
ifstream f("testpatterns");
while (!f.eof()) {
string query = getQuery(f);
ResType r1 = pr->execute(query);
ResType r2 = pr->plain_execute(query);
assert_s(r1.ok, "query failed: " + query);
assert_s(r2.ok, "query failed for plain: " + query);
assert_s(match(r1, r2), "plain and trained cryptdb give different results on query " + query);
LOG(test) << "MATCH!";
}
f.close();
cerr << "Test train passed\n";
*/
}
//do not change: has been used in creating the DUMPS for experiments
const uint64_t mkey = 113341234;
/*
static void __attribute__((unused))
evalImproveSummations(const TestConfig &tc)
{
string masterKey = BytesFromInt(mkey, AES_KEY_BYTES);
string host = tc.host;
string user = tc.user;
string db = tc.db;
string pwd = tc.pass;
cerr << "connecting to host " << host << " user " << user << " pwd " <<
pwd << " db " << db << endl;
EDBProxy * cl = new EDBProxy(host, user, pwd, db);
cl->setMasterKey(masterKey);
cl->VERBOSE = true;
cl->execute("CREATE TABLE test_table (id enc integer, name enc text)");
unsigned int no_inserts = 100;
unsigned int no_sums = 20;
for (unsigned int i = 0; i < no_inserts; i++) {
cl->execute(string("INSERT INTO test_table VALUES (") +
StringFromVal(i) + " , 'ana');");
}
startTimer();
for (unsigned int i = 0; i < no_sums; i++) {
cl->execute("SELECT sum(id) FROM test_table;");
}
double time = readTimer();
cerr << "time per sum: " << time/(1.0*no_sums) << " ms \n";
}
*/
/*
static void __attribute__((unused))
microEvaluate(const TestConfig &tc, int argc, char ** argv)
{
cout << "\n\n Micro Eval \n------------------- \n \n";
string masterKey = BytesFromInt(mkey, AES_KEY_BYTES);
EDBProxy * clsecure =
new EDBProxy(tc.host, tc.user, tc.pass, tc.db);
clsecure->setMasterKey(masterKey);
EDBProxy * clplain = new EDBProxy(tc.host, tc.user, tc.pass,
tc.db);
clsecure->VERBOSE = false;
clplain->VERBOSE = false;
clsecure->execute("CREATE TABLE tableeval (id integer, age integer);");
clplain->execute("CREATE TABLE tableeval (id integer, age integer);");
int nrInsertsSecure = 500;
int nrInsertsPlain = 1000;
int nrSelectsPlain = 2000;
int nrSelectsSecure = 3000;
int networkOverhead = 10; //10 ms latency per query
startTimer();
for (int i = 0; i < nrInsertsSecure; i++) {
unsigned int val1 = rand() % 10;
unsigned int val2 = rand() % 10;
string qq = "INSERT INTO tableeval VALUES ( " + strFromVal(val1) +
", " + strFromVal(val2) + ");";
clsecure->execute(qq);
}
double endTimer = (readTimer()/(1.0 * nrInsertsSecure));
printf("secure insertion: no network %6.6f ms with network %6.6f ms \n",
endTimer,
endTimer+networkOverhead);
startTimer();
for (int i = 0; i < nrInsertsPlain; i++) {
unsigned int val1 = rand() % 10;
unsigned int val2 = rand() % 10;
string qq = "INSERT INTO tableeval VALUES ( " + strFromVal(val1) +
", " + strFromVal(val2) + ");";
clplain->execute(qq);
}
endTimer = (readTimer()/(1.0 * nrInsertsPlain));
printf("plain insertion no network %6.6f ms with network %6.6f ms \n",
endTimer,
endTimer+networkOverhead);
startTimer();
for (int i = 0; i < nrSelectsSecure; i++) {
unsigned int val1 = rand() % 50;
string qq =
"SELECT tableeval.id FROM tableeval WHERE tableeval.id = " +
strFromVal(val1) + ";";
clsecure->execute(qq);
//unsigned int val2 = rand() % 50;
//qq = "SELECT tableeval.age FROM tableeval WHERE tableeval.age > " +
// strFromVal(val2) + ";";
clsecure->execute(qq);
}
endTimer = (readTimer()/(1.0 * nrSelectsSecure));
printf("secure selection no network %6.6f ms with network %6.6f ms \n",
endTimer,
endTimer+networkOverhead);
startTimer();
for (int i = 0; i < nrSelectsPlain; i++) {
unsigned int val1 = rand() % 50;
string qq =
"SELECT tableeval.id FROM tableeval WHERE tableeval.id = " +
strFromVal(val1) + ";";
clplain->execute(qq);
//unsigned int val2 = rand() % 50;
//qq = "SELECT tableeval.age FROM tableeval WHERE tableeval.age > " +
// strFromVal(val2) + ";";
clplain->execute(qq);
}
endTimer = (readTimer()/(1.0 * nrSelectsPlain));
printf("plain selection no network %6.6f ms with network %6.6f ms \n",
endTimer,
endTimer+networkOverhead);
clsecure->execute("DROP TABLE tableeval;");
clplain->execute("DROP TABLE tableeval;");
delete clsecure;
delete clplain;
}
*/
//at this point this function is mostly to figure our how binary data
// works..later will become a test
/*
void tester::testMarshallBinary() {
VERBOSE = true;
execute("CREATE TABLE peoples (id bytea);");
unsigned int len = 16;
unsigned char * rBytes = randomBytes(len);
cerr << " random Bytes are " << CryptoManager::marshallKey(rBytes) << "
\n";
cerr << " and marshalled they are " << marshallBinary(rBytes, len) <<
"\n";
string query = "INSERT INTO peoples VALUES ( " + marshallBinary(rBytes,
len) + " );";
execute(query);
PGresult * res = execute("SELECT id, octet_length(id) FROM
peoples;")->result;
cout << "repr " << PQgetvalue(res, 0, 0) << "\n";
execute("DROP TABLE peoples;");
}
*/
/*
//integration test
static void __attribute__((unused))
testEDBProxy(const TestConfig &tc)
{
cout << "\n\n Integration Queries \n------------------- \n \n";
string masterKey = BytesFromInt(mkey, AES_KEY_BYTES);
EDBProxy * cl = new EDBProxy(tc.host, tc.user, tc.pass, tc.db);
cl->setMasterKey(masterKey);
cl->VERBOSE = true;
cl->execute("CREATE TABLE people (id integer, name text, age integer);");
cl->execute("INSERT INTO people VALUES (34, 'raluca', 100);");
cl->execute("INSERT INTO people VALUES (35, 'alice', 20);");
cl->execute("INSERT INTO people VALUES (36, 'bob', 10);");
cl->execute("SELECT people.id, people.age, people.name FROM people;");
cl->execute("SELECT people.name FROM people WHERE people.age > 20;");
cl->execute(
"SELECT people.name, people.age FROM people WHERE people.name = 'alice' ;");
cl->execute("DROP TABLE people;");
delete cl;
cout << "\n------------------- \n Integration test succeeded \n\n";
}
*/
/*
static void
testPaillier(const TestConfig &tc, int ac, char **av)
{
int noTests = 100;
int nrTestsEval = 100;
string masterKey = randomBytes(AES_KEY_BYTES);
CryptoManager * cm = new CryptoManager(masterKey);
for (int i = 0; i < noTests; i++) {
int val = abs(rand() * 398493) % 12345;
cerr << "Encrypt " << val << "\n";
string ciph = cm->encrypt_Paillier(val);
//myPrint(ciph, CryptoManager::Paillier_len_bytes);
int dec = cm->decrypt_Paillier(ciph);
//cerr << "\n decrypt to: " << dec << "\n";
myassert(val == dec, "decrypted value is incorrect ");
}
string homCiph =
homomorphicAdd(homomorphicAdd(cm->encrypt_Paillier(123),
cm->encrypt_Paillier(234),
cm->getPKInfo()),
cm->encrypt_Paillier(1001), cm->getPKInfo());
myassert(cm->decrypt_Paillier(
homCiph) == 1358, "homomorphic property fails! \n");
cerr << "decrypt of hom " << cm->decrypt_Paillier(homCiph) <<
" success!! \n";
cerr << "Test Paillier SUCCEEDED \n";
cerr << "\n Benchmarking..\n";
string ciphs[nrTestsEval];
startTimer();
for (int i = 0; i < noTests; i++) {
int val = (i+1) * 10;
ciphs[i] = cm->encrypt_Paillier(val);
}
double res = readTimer();
cerr << "encryption takes " << res/noTests << " ms \n";
startTimer();
for (int i = 0; i < noTests; i++) {
int val = (i+1) * 10;
int dec = cm->decrypt_Paillier(ciphs[i]);
myassert(val == dec, "invalid decryption");
}
res = readTimer();
cerr << "decryption takes " << res/noTests << " ms \n";
}
static void
testUtils(const TestConfig &tc, int ac, char **av)
{
const char * query =
"SELECT sum(1), name, age, year FROM debug WHERE debug.name = 'raluca ?*; ada' AND a+b=5 ORDER BY name;";
LOG(test) << toString(parse(query, delimsStay, delimsGo, keepIntact), id_op);
}
static void __attribute__((unused))
createTables(string file, EDBProxy * cl)
{
ifstream createsFile(file);
if (createsFile.is_open()) {
while (!createsFile.eof()) {
string query = "";
string line = "";
while ((!createsFile.eof()) &&
(line.find(';') == string::npos)) {
createsFile >> line;
query = query + line;
}
if (line.length() > 0) {
cerr << query << "\n";
if (!cl->execute(query).ok) {
cerr << "FAILED on query " << query << "\n";
createsFile.close();
delete cl;
exit(1);
}
}
}
} else {
cerr << "error opening file " + file + "\n";
}
createsFile.close();
}
*/
static void __attribute__((unused))
convertQueries()
{
std::ifstream firstfile("eval/tpcc/client.sql");
std::ofstream secondfile("eval/tpcc/clientplain.sql");
std::string line;
int nr= 0;
std::string transac = "";
if (!firstfile.is_open()) {
std::cerr << "cannot open input file \n";
return;
}
if (!secondfile.is_open()) {
std::cerr << "cannot open a second file \n";
return;
}
while (!firstfile.eof() )
{
getline (firstfile,line);
if (line.length() <= 1 ) {
continue;
}
line = line + ";";
//extract transaction number
std::string no = line.substr(0, line.find('\t'));
std::cerr << "no transac is " << no << "\n";
line = line.substr(no.length()+1, line.length() - no.length()+1);
if (no.compare(transac) != 0) {
if (transac.length() > 0) {
secondfile << "commit; \n" << "begin; \n";
} else {
secondfile << "begin; \n";
}
transac = no;
}
size_t index = 0;
while (line.find(".", index) != std::string::npos) {
size_t pos = line.find(".", index);
index = pos+1;
if (line[pos+1] >= '0' && line[pos+1] <= '9') {
while ((line[pos]!=' ') && (line[pos] != ',') &&
(line[pos] != '\'')) {
line[pos] = ' ';
pos++;
}
}
}
while (line.find(">=") != std::string::npos) {
size_t pos = line.find(">=");
line[pos+1] = ' ';
//TRACE SPECIFIC
pos = pos + 3;
int nr2 = 0;
size_t oldpos = pos;
while (line[pos] != ' ') {
nr2 = nr2 * 10 + (line[pos]-'0');
pos++;
}
nr2 = nr2 - 20;
std::string replacement = strFromVal((uint32_t)nr2) + " ";
line.replace(oldpos, 9, replacement);
}
while (line.find("<=") != std::string::npos) {
size_t pos = line.find("<");
line[pos+1] = ' ';
}
while (line.find(" -") != std::string::npos) {
size_t pos = line.find(" -");
line[pos+1] = ' ';
}
secondfile << line << "\n";
nr++;
}
std::cerr << "there are " << nr << " queries \n";
firstfile.close();
secondfile.close();
}
/*
static void __attribute__((unused))
test_train(const TestConfig &tc)
{
cerr << "training \n";
string masterKey = BytesFromInt(mkey, AES_KEY_BYTES);
EDBProxy * cl = new EDBProxy(tc.host, tc.user, tc.pass, tc.db);
cl->setMasterKey(masterKey);
cl->VERBOSE = true;
cl->dropOnExit = true;
cl->train("eval/tpcc/sqlTableCreates");
cl->train("eval/tpcc/querypatterns.txt");
cl->train("eval/tpcc/index.sql");
cl->train_finish();
cl->create_trained_instance();
delete cl;
}
*/
/*
void createIndexes(string indexF, EDBProxy * cl) {
ifstream tracefile(indexF);
string query;
if (tracefile.is_open()) {
while (!tracefile.eof()){
getline(tracefile, query);
if (query.length() > 1) {
edb_result * res =
cl->execute(query);
if (res == NULL) {
cerr << "FAILED on query " << query <<
"\n";
delete cl;
tracefile.close();
exit(1);
}
}
}
}
tracefile.close();
}
void convertDump() {
ifstream fileIn("eval/tpcc/dumpcustomer.sql");
ofstream fileOut("eval/tpcc/dumpcustomer");
string query;
int index = 0;
if (fileIn.is_open()) {
while (!fileIn.eof()){
getline(fileIn, query);
while (query.find("`")!=string::npos) {
int pos = query.find("`");
query[pos] = ' ';
}
if (query.length() > 1) {
fileOut << query << "\n";
index++;
}
}
}
cerr << "there are " << index << "entries \n";
fileIn.close();
fileOut.close();
}
*/
/*
const string createFile = "eval/tpcc/sqlTableCreates";
const string queryTrainFile = "eval/tpcc/querypatterns.txt";
const string dumpFile = "eval/tpcc/pieces/dump";
const string queryFilePrefix = "eval/tpcc/";
const string indexFile = "eval/tpcc/index.sql";
const int nrDataToLoad = 100000; //how much the threads load in total
const bool verbose = true;
const bool isSecureLoad = true;
const int logFrequency = 500;
void runTrace(const char * suffix, int nrQueriesToRun, int isSecure) {
struct timeval tvstart, tvend;
unsigned char * masterKey = BytesFromInt(mkey, AES_KEY_BYTES);
EDBProxy * cl;
if (isSecure) {
cl = new EDBProxy("cryptdb", masterKey);
} else {
cl = new EDBProxy(tc.db);
}
cl->VERBOSE = verbose;
cerr << "prepare \n";
cl->nosubmit = true;
createTables(createFile, cl);
runForSteps(queryTrainFile, 100, cl);
createIndexes(indexFile, cl);
cl->nosubmit = false;
gettimeofday(&tvstart, NULL);
runForSteps(queryFilePrefix+suffix, nrQueriesToRun, cl);
gettimeofday(&tvend,NULL);
cerr << "all trace took " << ((tvend.tv_sec - tvstart.tv_sec)*1.0 +
(tvend.tv_usec - tvstart.tv_usec)/1000000.0) << "s \n";
}
*/
static std::string __attribute__((unused))
suffix(int no)
{
int fi = 97 + (no/26);
int se = 97 + (no%26);
std::string first = std::string("") + (char)fi;
std::string second = std::string("") + (char)se;
std::string res = first + second;
return res;
}
/*
void tester::loadData(EDBProxy * cl, string workload, int logFreq) {
ifstream dataFile(workload);
ofstream outFile(workload+"answer");
if ((!dataFile.is_open()) || (!outFile.is_open())) {
cerr << "could not open file " << workload << "or the answer
file " << " \n";
exit(1);
}
string query;
int index = 0;
while (!dataFile.eof()) {
getline(dataFile, query);
if (query.length() > 0) {
if (index % logFreq == 0) {
cerr << "load entry " << index << " from " <<
workload << "\n" ;// query " << query <<
"\n";
}
*/
/*edb_result * res = cl->execute(query);
if (index % logFreq== 0) {//cout << "executed\n";
}
if (res == NULL) {
cerr << workload + "offensive insert query "
<< query << " \n";
while (res == NULL) {
cerr << "retrying \n";
sleep(10);
res = cl->execute(query);
//delete cl;
//dataFile.close();
//exit(1);
}
}*//*
list<string> ress;
try {
ress =
tcl->rewriteEncryptQuery(query);
} catch (CryptDBError se) {
cerr << "CryptDB error " << se.msg << "\n";
exit(1);
}
outFile << ress.front() << "\n";
}
index++;
}
outFile.close();
dataFile.close();
return;
}
bool isbegin(string s) {
if (s.compare("begin;") == 0){
return true;
}
if (s.compare("begin; ") == 0){
return true;
}
return false;
}
bool iscommit(string s) {
if (s.compare("commit;") == 0){
return true;
}
if (s.compare("commit; ") == 0){
return true;
}
return false;
}
void runTrace(EDBProxy * cl, int logFreq, string workload, bool isSecure,
bool hasTransac,
int & okinstrCount, int & oktranCount, int & totalInstr, int &
totalTran) {
ifstream tracefile(workload);
string query;
if (!tracefile.is_open()) {
cerr << "cannot open " << workload << "\n";
}
int count = 0;//nr of failed instructions
int index = 0; //the number of query execute currently
//counts for the number of instructions or transactions succesfully
run
okinstrCount = 0;
oktranCount = 0;
totalTran = 0;
totalInstr = 0;
bool tranAborted = false;
while (!tracefile.eof()) {
getline(tracefile, query);
if (query.length() < 1) {
continue;
}
index++;
if (index % logFreq == 0) {cerr << workload << " " <<
index << "\n";}
if (!hasTransac) {
if (isSecure) {
edb_result * res =
cl->execute(query);
if (res == NULL) {
count++;
}
} else {
PGresult * res =
cl->plain_execute(query);
if (!((PQresultStatus(res) == PGRES_TUPLES_OK)
|| (PQresultStatus(res) ==
PGRES_COMMAND_OK))) {
count++;
}
}
} else {
if (isbegin(query)) {
tranAborted = false;
totalTran++;
}
if (tranAborted) {
continue;
}
bool instrOK = true;
if (isSecure) {
edb_result * res =
cl->execute(query);
if (res == NULL) {
instrOK = false;
}
} else {
PGresult * res =
cl->plain_execute(query);
if (!((PQresultStatus(res) ==
PGRES_TUPLES_OK) ||
(PQresultStatus(res) ==
PGRES_COMMAND_OK))) {
instrOK = false;
}
}
if (!instrOK) {
PGresult * res =
cl->plain_execute("abort;");
if (!((PQresultStatus(res) ==
PGRES_TUPLES_OK) ||
(PQresultStatus(res) ==
PGRES_COMMAND_OK))) {
cerr <<
workload <<
":
returning
bad status
even on
abort,
exiting ;";
tracefile.close();
exit(1);
}
tranAborted = true;
cerr << "aborting curr tran \n";
} else {
if (iscommit(query)) {
oktranCount++;
//cerr << "oktranCount becomes
" << oktranCount << " \n";
}
okinstrCount++;
}
}
}
totalInstr = index;
tracefile.close();
return;
}
void simpleThroughput(int noWorkers, int noRepeats, int logFreq, int
totalLines, string dfile, bool isSecure, bool hasTransac) {
cerr << "throughput benchmark \n";
*/
// int res = system("rm eval/pieces/*");
/*
ifstream infile(dfile);
int linesPerWorker = totalLines / noWorkers;
string query;
if (!infile.is_open()) {
cerr << "cannot open " + dfile << "\n";
exit(1);
}
//prepare files
for (int i = 0; i < noWorkers; i++) {
string workload = string("eval/pieces/piece") + suffix(i);
ofstream outfile(workload);
if (!outfile.is_open()) {
cerr << "cannot open file " << workload << "\n";
infile.close();
exit(1);
}
getline(infile, query);
if (hasTransac && (!isbegin(query))) {
outfile << "begin; \n";
}
for (int j = 0; j < linesPerWorker; j++) {
outfile << query << "\n";
if (j < linesPerWorker-1) {getline(infile, query);}
}
if (hasTransac && (!iscommit(query))) {
outfile << "commit; \n";
}
outfile.close();
//we need to concatenate the outfile with itself noRepeats
times
res = system("touch temp;");
for (int j = 0; j < noRepeats; j++) {
res = system(string("cat temp ") + workload +
string(" > temp2;"));
res = system("mv temp2 temp");
}
res = system("mv temp " + workload);
}
infile.close();
res = system("rm eval/pieces/result;");
res = system("touch eval/pieces/result;");
ofstream resultFile("eval/pieces/result");
ifstream resultFileIn;
if (!resultFile.is_open()) {
cerr << "cannot open result file \n";
exit(1);
}
timeval starttime, endtime;
int childstatus;
int index;
int i;
pid_t pids[noWorkers];
double interval, querytput, querylat, trantput, tranlat;
int allInstr, allInstrOK, allTran, allTranOK;
for (i = 0; i < noWorkers; i++) {
index = i;
pid_t pid = fork();
if (pid == 0) {
goto dowork;
} else if (pid < 0) {
cerr << "failed to fork \n";
exit(1);
} else { // in parent
pids[i] = pid;
}
}
//parent
for (i = 0; i < noWorkers; i++) {
if (waitpid(pids[i], &childstatus, 0) == -1) {
cerr << "there were problems with process " << pids[i]
<< "\n";
}
}
resultFile.close();
resultFileIn.open("eval/pieces/result", ifstream::in);
if (!resultFileIn.is_open()) {
cerr << "cannot open results file to read\n";
exit(1);
}
querytput = 0; querylat = 0; trantput = 0; tranlat = 0;
allInstr = 0; allInstrOK = 0; allTran = 0; allTranOK = 0;
for (i = 0; i < noWorkers; i++) {
double currquerytput, currquerylat, currtrantput, currtranlat;
int currtotalInstr, currokInstr, currtotalTran, currokTran;
if (!hasTransac) {
resultFileIn >> index; resultFileIn >> interval;
resultFileIn >> currquerytput; resultFileIn >>
currquerylat;
cerr << index << " " << interval << " sec " <<
currquerytput << " queries/sec " << currquerylat <<
" secs/query \n";
querytput = querytput + currquerytput;
querylat = querylat + currquerylat; }
else {
resultFileIn >> index;
resultFileIn >> interval; resultFileIn >>
currtrantput; resultFileIn >> currquerytput;
resultFileIn >> currtranlat; resultFileIn >>
currquerylat; resultFileIn >> currtotalInstr;
resultFileIn >> currokInstr; resultFileIn >>
currtotalTran; resultFileIn >> currokTran;
querytput +=currquerytput;
querylat +=currquerylat;
trantput += currtrantput;
tranlat += currtranlat;
allInstr += currtotalInstr;
allInstrOK += currokInstr;
allTran += currtotalTran;
allTranOK += currokTran;
cerr << "worker " << i << " interval " << interval <<
" okInstr " << currokInstr << " okTranCount " <<
currokTran << " totalInstr " << currtotalInstr << "
totalTran " << currtotalTran << "\n";
}
}
if (!hasTransac) {
cerr <<"overall: throughput " << querytput << " queries/sec
latency " << querylat/noWorkers << " sec/query \n";
} else {
querylat = querylat / noWorkers;
tranlat = tranlat / noWorkers;
if (isSecure) {
cerr << "secure: ";
} else {
cerr << "plain: ";
}
cerr << " querytput " << querytput << " querylat " << querylat
<< " trantput " << trantput << " tranlat " << tranlat << "
allInstr " << allInstr << " allInstrOK " << allInstrOK << "
tran failed" << allTran-allTranOK << " allTransacOK " <<
allTranOK << " \n";
}
resultFileIn.close();
return;
//children:
dowork:
EDBProxy * cl;
if (isSecure) {
unsigned char * masterKey = BytesFromInt(mkey,
AES_KEY_BYTES);
cl = new EDBProxy("cryptdb", masterKey);
cl->train("schema");
cl->train("queries");
cl->train_finish();
} else {
cl = new EDBProxy(tc.db);
}
string workload = string("eval/pieces/piece") + suffix(index);
//execute on the workload
cerr << "in child workload file <" << workload << "> \n";
cerr << "value of index is " << index << "\n";
int okInstrCount, okTranCount, totalInstr, totalTran;
gettimeofday(&starttime, NULL);
runTrace(cl, logFreq, workload, isSecure, hasTransac, okInstrCount,
okTranCount, totalInstr, totalTran);
//now we need to start the workers
gettimeofday(&endtime, NULL);
interval = 1.0 * timeInSec(starttime, endtime);
if (!hasTransac) {
querytput = linesPerWorker * noRepeats * 1.0 / interval;
querylat = interval / (linesPerWorker * noRepeats);
resultFile << index << " " << interval << " " << querytput <<
" " << querylat << "\n";
} else { //report workerid timeinterval trantput querytput tranlate
querylate totalInstr okInstr totalTRan okTran
myassert(noRepeats == 1, "repeats more than one,transactions
fail automatically\n");
double trantput = okTranCount * 1.0 / interval;
double querytput = okInstrCount*1.0/interval;
double tranlat = interval*1.0/okTranCount;
double querylat = interval*1.0/okInstrCount;
resultFile << index << " " << interval << " " << trantput << "
" << querytput << " " << tranlat << " " << querylat << " "
<< totalInstr << " " << okInstrCount << " " << totalTran <<
" " << okTranCount << "\n";
}
delete cl;
return;
}
void createInstance() {
unsigned char * masterKey = BytesFromInt(mkey, AES_KEY_BYTES);
EDBProxy * cl = new EDBProxy("cryptdb", masterKey);
cl->train("eval/tpcc/sqlTableCreates");
cl->train("eval/tpcc/querypatterns.txt");
cl->train("eval/tpcc/index.sql");
cl->train_finish();
cl->create_trained_instance(true);
EDBProxy * plaincl = new EDBProxy(tc.db);
int res = system("psql < eval/tpcc/sqlTableCreates");
res = system("psql < eval/tpcc/index.sql");
delete cl;
delete plaincl;
}
void parallelLoad(int noWorkers, int totalLines, int logFreq, string
dfile, int workeri1, int workeri2) {
cerr << "Parallel loading from " << dfile << ". \n";
unsigned char * masterKey = BytesFromInt(mkey, AES_KEY_BYTES);
tester t = tester("cryptdb", masterKey);
int res = system("mkdir eval/pieces;");
*/
// res = system("rm eval/pieces/*");
/*
string splitComm = "split -l " +
strFromVal((uint32_t)(totalLines/noWorkers)) + " -a 2 " + dfile +
" eval/pieces/piece";
cerr << "split comm " << splitComm << "\n";
res = system(splitComm);
myassert(res == 0, "split failed");
EDBProxy * cl = new EDBProxy("cryptdb", masterKey);
cl->train("eval/tpcc/sqlTableCreates");
cl->train("eval/tpcc/querypatterns.txt");
cl->train("eval/tpcc/index.sql");
cl->train_finish();
cl->create_trained_instance(false);
int index = 0;
for (int i = workeri1; i <= workeri2; i++) {
index = i;
pid_t pid = fork();
if (pid == 0) {
goto dowork;
} else if (pid < 0) {
cerr << "failed to fork \n";
exit(1);
}
}
//parent
return;
dowork:
string workload = string("eval/pieces/piece") + suffix(index);
//execute on the workload
cerr << "in child workload file <" << workload << "> \n";
cerr << "value of index is " << index << "\n";
t.loadData(cl, workload, logFreq);
}
void executeQueries(EDBProxy * cl, string workload, string resultFile, int
timeInSecs, int logFreq) {
ifstream tracefile(workload);
string query;
struct timeval tvstart, tvend;
if (!tracefile.is_open()) {
cerr << "cannot open " << workload << "\n";
exit(1);
}
gettimeofday(&tvstart, NULL);
gettimeofday(&tvend,NULL);
int index = 0;
while (timeInSec(tvstart, tvend) < timeInSecs) {
while (!tracefile.eof()) {
getline(tracefile, query);
edb_result * res = cl->execute(query);
index++;
if (index % logFreq == 0) {cerr << index << "\n";}
if (res == NULL) {
cerr << "FAILED on query " << query << "\n";
cerr << "query no " << index << "\n";
delete cl;
tracefile.close();
exit(1);
}
if (index % 100 == 0) {
gettimeofday(&tvend, NULL);
if (timeInSec(tvstart, tvend) >= timeInSecs) {
goto wrapup;
}
}
}
gettimeofday(&tvend,NULL);
}
wrapup:
tracefile.close();
ofstream resFile(getCStr(resultFile));
if (!resFile.is_open()) {
cerr << "cannot open result file " << resultFile << "\n";
exit(-1);
}
resFile << index << "\n";
resFile << timeInSec(tvstart, tvend) << "\n";
resFile.close();
}
void throughput(int noClients, int totalLines, int timeInSecs, int logFreq,
string queryFile, bool isSecure) {
//prepare files
int res = system("mkdir eval/queries;");
*/
// res = system("rm eval/queries/*");
/*
string splitComm = "split -l " +
strFromVal((uint32_t)(totalLines/noClients)) + " -a 1 " +
queryFile + " eval/queries/piece";
cerr << "split comm " << splitComm << "\n";
res = system(getCStr(splitComm));
myassert(res == 0, "split failed");
EDBProxy * cl;
if (isSecure) {
unsigned char * masterKey = BytesFromInt(mkey,
AES_KEY_BYTES);
cl = new EDBProxy("cryptdb", masterKey);
cl->train("eval/tpcc/sqlTableCreates");
cl->train("eval/tpcc/querypatterns.txt");
cl->train("eval/tpcc/index.sql");
cl->train_finish();
}
else {
cl = new EDBProxy(tc.db);
}
int index = 0;
pid_t pids[noClients];
string resultFile[noClients];
int i, childstatus;
pid_t pid;
double throughput;
for (i = 0; i < noClients; i++) {
resultFile[i] = "eval/queries/answer" + (char)('a'+i);
index = i;
pid = fork();
if (pid == 0) {
goto dowork;
} else if (pid < 0) {
cerr << "failed to fork \n";
exit(1);
} else {
//parent
pids[i] = pid;
}
}
//parents
for (i = 0; i < noClients; i++) {
if (waitpid(pids[i], &childstatus, 0) == -1) {
cerr << "there were problems with process " << pids[i]
<< "\n";
}
}
//collect results and compute throughput
throughput = 0;
for (int i = 0; i < noClients; i++) {
string resfile = string("eval/queries/answer")+char('a'+i);
ifstream result(getCStr(resfile));
int count;
double secs;
result >> count;
result >> secs;
cerr << "worker i processed " << count << " in " << secs << "
secs \n";
throughput += (count*1.0/secs);
}
cerr << "overall throughput " << throughput << "\n";
return;
dowork: //child
string workload = string("eval/queries/piece") + (char)('a' + index);
//execute on the workload
cerr << "in child workload file <" << workload << "> \n";
cerr << "value of index is " << index << "\n";
executeQueries(cl, workload, resultFile[index], timeInSecs, logFreq);
return;
}
void parse() {
ifstream f("queries");
ofstream o("queries2");
string q;
while (!f.eof()) {
getline(f, q);
if (q.length() > 0) {
o << q << ";\n";
}
}
f.close();
o.close();
}
void simple() {
unsigned char * masterKey = BytesFromInt(mkey, AES_KEY_BYTES);
EDBProxy * cl = new EDBProxy("cryptdb", masterKey);
cl->train("schema");
cerr << "done with creates \n";
cl->train("queries");
cl->train_finish();
// cl->create_trained_instance();
ifstream f("insertslast");
if (!f.is_open()) {
cerr << "cannot open f \n";
exit(1);
}
cl->ALLOW_FAILURES = true;
int log = 2000;
int index = 0;
string q;
while (!f.eof()) {
getline(f, q);
index++;
if (index % log == 0) {cerr << index << "\n";}
if (q.length() == 0) {
continue;
}
cl->execute(q);
}
cerr << "done \n";
return;
}
void latency(string queryFile, int maxQueries, int logFreq, bool isSecure,
int isVerbose) {
*/
/*ofstream outputnew("cleanquery"); //DO
if (!outputnew.is_open()) { //DO
cerr << "cannot open cleanquery file \n";
exit(1);
}
*/
/* cerr << "starting \n";
struct timeval tvstart;
struct timeval tvend;
EDBProxy * cl;
if (isSecure) {
unsigned char * masterKey = BytesFromInt(mkey,
AES_KEY_BYTES);
cl = new EDBProxy("cryptdb", masterKey);
if (isVerbose) {
cl->VERBOSE = true;
} else {
cl->VERBOSE = false;
}
cl->train("eval/tpcc/sqlTableCreates");
cerr << "done with creates \n";
cl->train("eval/tpcc/querypatterns.txt");
cl->train_finish();
cerr << "done \n";
return;
}
else {
cl = new EDBProxy(tc.db);
}
if (isVerbose) {
cl->VERBOSE = true;
} else {
cl->VERBOSE = false;
}
ifstream tracefile(getCStr(queryFile));
if (!tracefile.is_open()) {
cerr << "cannot open file " << queryFile << "\n";
exit(-1);
}
gettimeofday(&tvstart, NULL);
string query;
int index = 0;
while (!tracefile.eof()) {
getline(tracefile, query);
if (query.size() == 0) {
continue;
}
*/ /* try {
PGresult * res =
(*cl)->plain_execute(query);//DO
ExecStatusType est = PQresultStatus(res);
if ((est == PGRES_COMMAND_OK) || (est ==
(*PGRES_TUPLES_OK))) {
outputnew << query << "\n";
} else {
cl->plain_execute("abort;");
}
*/
/* cl->rewriteEncryptQuery(query);
//cerr << query << "\n";
//cerr << resQuery.front() << "\n";
} catch (CryptDBError se) {
cerr << se.msg << "\n aborting \n";
return;
}
if (index % logFreq == 0) {cerr << index << "\n";}
index++;
if (index == maxQueries) {
break;
}
}
//outputnew.close(); //DO
gettimeofday(&tvend,NULL);
tracefile.close();
cerr << "file " << queryFile << " overall took " << timeInSec(tvstart,
tvend) << " each statement took " << (timeInSec(tvstart,
tvend)/index*1.0)*1000.0 << " ms \n";
}
*/
/*
static void
encryptionTablesTest(const TestConfig &tc, int ac, char **av)
{
EDBProxy * cl =
new EDBProxy(tc.host, tc.user, tc.pass, tc.db);
cl->setMasterKey(randomBytes(AES_KEY_BYTES));
int noHOM = 100;
int noOPE = 100;
cl->VERBOSE = true;
if (!cl->execute("CREATE TABLE try (age integer);").ok)
return;
struct timeval starttime, endtime;
gettimeofday(&starttime, NULL);
cl->createEncryptionTables(noHOM, noOPE);
gettimeofday(&endtime, NULL);
cerr << "time per op" <<
timeInSec(starttime, endtime)*1000.0/(noHOM+noOPE) << "\n";
if (!cl->execute("INSERT INTO try VALUES (4);").ok) return;
if (!cl->execute("INSERT INTO try VALUES (5);").ok) return;
if (!cl->execute("INSERT INTO try VALUES (5);").ok) return;
if (!cl->execute("INSERT INTO try VALUES (5);").ok) return;
if (!cl->execute("INSERT INTO try VALUES (5);").ok) return;
if (!cl->execute("INSERT INTO try VALUES (5);").ok) return;
if (!cl->execute("INSERT INTO try VALUES (5);").ok) return;
if (!cl->execute("INSERT INTO try VALUES (10000001);").ok) return;
if (!cl->execute("SELECT age FROM try WHERE age > 1000000;").ok) return;
if (!cl->execute("DROP TABLE try;").ok) return;
delete cl;
}
*/
static void
testParseAccess(const TestConfig &tc, int ac, char **av)
{
std::cerr << "testParseAccess uses old EDBProxy -- not run"
<< std::endl;
/*
EDBProxy * cl = new EDBProxy(tc.host, tc.user, tc.pass, tc.db);
cl->setMasterKey(BytesFromInt(mkey, AES_KEY_BYTES));
cl->VERBOSE = true;
cl->execute(
"CREATE TABLE test (gid integer, t text encfor gid, f integer encfor mid, mid integer);");
string q = "INSERT INTO test VALUES (3, 'ra', 5, 4);";
cerr << q;
list<string> query = parse(q, delimsStay, delimsGo, keepIntact);
TMKM tmkm;
QueryMeta qm;
assert_s(false, "test no longer valid because of interface change ");
//cl->getEncForFromFilter(INSERT, query, tmkm, qm); <-- no longer valid
map<string, string>::iterator it = tmkm.encForVal.begin();
while (it != tmkm.encForVal.end()) {
cout << it->first << " " << it->second << "\n";
it++;
}
q = "SELECT * FROM test WHERE gid = 3 AND mid > 4 AND t = 'ra';";
cerr << q;
query = parse(q, delimsStay, delimsGo, keepIntact);
//cl->getEncForFromFilter(SELECT, query, tmkm, qm); <-- no longer valid
// due to interface change
it = tmkm.encForVal.begin();
while (it != tmkm.encForVal.end()) {
cout << it->first << " " << it->second << "\n";
it++;
}
cl->execute("DROP TABLE test;");
*/
}
static void
autoIncTest(const TestConfig &tc, int ac, char **av)
{
/*
string masterKey = BytesFromInt(mkey, AES_KEY_BYTES);
string host = tc.host;
string user = tc.user;
string db = tc.db;
string pwd = tc.pass;
cerr << "connecting to host " << host << " user " << user << " pwd " <<
pwd << " db " << db << endl;
EDBProxy * cl = new EDBProxy(host, user, pwd, db);
cl->setMasterKey(masterKey);
cl->VERBOSE = true;
cl->plain_execute("DROP TABLE IF EXISTS t1, users;");
assert_res(cl->execute(
"CREATE TABLE t1 (id integer, post encfor id det text, age encfor id ope bigint);"),
"failed");
assert_res(cl->execute(
"CREATE TABLE users (id equals t1.id integer, username givespsswd id text);"),
"failed");
assert_res(cl->execute(
"INSERT INTO " psswdtable
" VALUES ('alice', 'secretalice');"),
"failed to log in user");
assert_res(cl->execute(
"DELETE FROM " psswdtable
" WHERE username = 'al\\'ice';"),
"failed to logout user");
assert_res(cl->execute(
"INSERT INTO " psswdtable
" VALUES ('alice', 'secretalice');"),
"failed to log in user");
assert_res(cl->execute(
"INSERT INTO users VALUES (1, 'alice');"),
"failed to add alice in users table");
ResType rt = cl->execute(
"INSERT INTO t1 (post, age) VALUES ('A you go', 23);");
assert_s(rt.ok && rt.names[0].compare("cryptdb_autoinc") == 0, "fieldname is not autoinc");
assert_s(rt.ok && rt.rows[0][0].data == "1", "autoinc not correct1");
rt = cl->execute(
"INSERT INTO t1 (post, age) VALUES ('B there you go', 23);");
assert_s(rt.ok && rt.rows[1][0].data == "2", "autoinc not correct2");
rt = cl->execute("INSERT INTO t1 VALUES (3, 'C there you go', 23);");
cerr << "result is " << rt.rows[0][0].to_string() << "\n";
assert_s(rt.ok && rt.rows[0][0].data == "3", "autoinc not correct3");
rt = cl->execute(
"INSERT INTO t1 (post, age) VALUES ( 'D there you go', 23);");
assert_s(rt.ok && rt.rows[0][0].data == "4", "autoinc not correct4");
*/
//delete cl;
}
typedef struct Stats {
int total_queries;
int queries_failed;
double mspassed;
int worker;
Stats() {
total_queries = 0;
queries_failed = 0;
mspassed = 0.0;
worker = 0;
}
} Stats;
//static Stats * myStats;
/*
* - if execQuery is true, the query is sent to the DB
* - if encryptQuery is true, the query is being rewritten
* - if conn != NULL, use this connection to send queries to the DBMS
* - if outputfile != "", log encrypted query here
* - if !allowExecFailures, we stop on failed queries sent to the DBMS
* - logFreq indicates frequency of logging
* - if execEncQuery is true the query is execute via cryptdb (query encrypted and results decrypted), the previous flags
* are not taken into account any more
*/
/*
static void
runQueriesFromFile(EDBProxy * cl, string queryFile, bool execQuery, bool encryptQuery, Connect * conn,
string outputfile, bool allowExecFailures, bool execEncQuery, Stats * stats, int logFreq)
throw (CryptDBError)
{
cerr << "runQueriesFromFile uses old EDBProxy -- not run" << endl;
ifstream infile(queryFile);
ofstream * outfile = NULL;
assert_s(infile.is_open(), "cannot open file " + queryFile);
assert_s(!execQuery || conn, "conn is null, while execQuery is true");
bool outputtranslation = false;
if (outputfile != "") {
outfile = new ofstream(outputfile);
outputtranslation = true;
}
string query;
list<string> queries;
Timer t;
while (!infile.eof()) {
query = getQuery(infile);
if (query.length() == 0) {
continue;
}
if (!execEncQuery) {
if (encryptQuery) {
try{
bool temp;
queries = cl->rewriteEncryptQuery(query+";", temp);
} catch(...) {
cerr << "query " << query << "failed";
cerr << "worker exits\n";
exit(-1);
}
} else {
queries = list<string>(1, query);
}
stats->total_queries++;
if (stats->total_queries % logFreq == 0) {
cerr << queryFile << ": " << stats->total_queries << " so far\n";
}
for (list<string>::iterator it = queries.begin(); it != queries.end(); it++) {
if (outputtranslation) {
*outfile << *it << "\n";
}
if (execQuery) {
bool outcome = conn->execute(*it);
if (allowExecFailures) {
if (!outcome) {
stats->queries_failed++;
}
} else {
assert_s(outcome, "failed to execute query " + *it);
}
}
}
} else {
ResType r = cl->execute(query);
assert_s(r.ok, "query failed");
//cerr << "results returned " << r.rows.size() << "\n";
stats->total_queries++;
}
}
stats->mspassed = t.lap_ms();
infile.close();
if (outputtranslation) {
outfile->close();
}
}
*/
/*static void
dotrain(EDBProxy * cl, string createsfile, string querypatterns, string exec) {
cl->execute(string("train ") + " 1 " + createsfile + " " + querypatterns + " " + exec);
}
static string * workloads;
static string resultFile;
static string
fixedRepr(int i) {
string s = StringFromVal(i);
while (s.length() < 5) {
s = "0" + s;
}
return s;
}
static void
assignWork(string queryfile, int noWorkers, int totalLines, int noRepeats, bool split) {
int blah = system("mkdir pieces");
assert_s(system("rm -f pieces") >= 0, "problem when removing pieces");
if (false) {
LOG(test) << blah;
}
ifstream infile(queryfile);
workloads = new string[noWorkers];
int linesPerWorker = totalLines / noWorkers;
int linesPerLastWorker = totalLines - (noWorkers-1)*linesPerWorker;
string query;
if (!infile.is_open()) {
cerr << "cannot open " + queryfile << "\n";
exit(1);
}
//prepare files
for (int i = 0; i < noWorkers; i++) {
string workload;
if (!split) {
workload = queryfile + fixedRepr(i);
} else {
workload = string("pieces/piece") + fixedRepr(i);
ofstream outfile(workload);
cerr << "creating worker workload " << workload << "\n";
if (!outfile.is_open()) {
cerr << "cannot open file " << workload << "\n";
infile.close();
exit(1);
}
int lines = 0;
if (i == noWorkers-1) {
lines = linesPerLastWorker;
} else {
lines = linesPerWorker;
}
for (int j = 0; j < lines; j++) {
getline(infile, query);
if (query != "") {
outfile << query << "\n";
}
}
outfile.close();
}
workloads[i] = workload;
//we need to concatenate the outfile with itself noRepeats
if (noRepeats > 1) {
assert_s(system("touch temp;") >= 0, "problem when creating temp");
for (int j = 0; j < noRepeats; j++) {
assert_s(system((string("cat temp ") + workload + " > temp2;").c_str()) >= 0, "problem when cat");
assert_s(system("mv temp2 temp") >= 0, "problem when moving");
}
assert_s(system(("mv temp " + workload).c_str()) >= 0, "problem when moving");
}
}
infile.close();
}
static void __attribute__((noreturn))
workerFinish() {
ofstream resFile(resultFile+StringFromVal(myStats->worker));
assert_s(resFile.is_open(), " could not open file " + resultFile);
resFile << myStats->worker << " " << myStats->queries_failed << " " << myStats->total_queries << " "
<< myStats->mspassed << "\n";
resFile.close();
string output = "worker " + StringFromVal(myStats->worker) + " queriesfailed " + StringFromVal(myStats->queries_failed) + " totalqueries "
+ StringFromVal(myStats->total_queries)+ " mspassed " + StringFromVal((int)myStats->mspassed) + "\n";
cerr << output;
exit(EXIT_SUCCESS);
}
//function is called when a worker must terminate
//it outputs its statistics and exits
static void __attribute__((noreturn))
signal_handler(int signum) {
//sleep for random interval to avoid races at writing out results
string res = "worker " + StringFromVal(myStats->worker) + " received signal \n";
cerr << res;
workerFinish();
}
static void
workerJob(EDBProxy * cl, int index, const TestConfig & tc, int logFreq) {
string workload = workloads[index];
//execute on the workload
cerr << "in child workload file <" << workload << "> \n";
Timer t;
myStats = new Stats();
myStats->worker = index;
Connect * conn = new Connect(tc.host, tc.user, tc.pass, tc.db, tc.port);
runQueriesFromFile(cl, workload, true, false, conn, "", true, false, myStats, logFreq);
cerr << "Done on " << workload << "\n";
myStats->mspassed = t.lap_ms();
workerFinish();
}
static void runExp(EDBProxy * cl, int noWorkers, const TestConfig & tc, int logFreq) {
assert_s(system("rm -f pieces/result") >= 0, "problem removing pieces/result");
assert_s(system("touch pieces/result;") >= 0, "problem creating pieces/result");
resultFile = "pieces/exp_result";
assert_s(signal(SIGTERM, signal_handler) != SIG_ERR ,"signal could not set the handler");
int childstatus;
int index;
int i;
pid_t pids[noWorkers];
double interval, querytput, querylat;
int allQueries, allQueriesOK;
for (i = 0; i < noWorkers; i++) {
index = i;
pid_t pid = fork();
if (pid == 0) {
workerJob(cl, index, tc, logFreq);
} else if (pid < 0) {
cerr << "failed to fork \n";
exit(1);
} else { // in parent
pids[i] = pid;
}
}
//wait until any child finishes
pid_t firstchild = waitpid(-1, &childstatus,0);
assert_s(WIFEXITED(childstatus), "the first child returning terminated abnormally\n");
cerr << "One child finished, stopping all other children..\n";
//signal the other children to stop
for (int j = 0; j < noWorkers; j++) {
if (pids[j] != firstchild) {
assert_s(kill(pids[j], SIGTERM) == 0, "could not send signal to child " + StringFromVal(j));
}
}
//now wait and make sure all finished
for (int j = 0; j < noWorkers; j++) {
if (pids[j] != firstchild) {
if (waitpid(pids[j], &childstatus, 0) == -1) {
cerr << "there were problems with process " << j << "\n";
}
}
}
interval = 0.0;
querytput = 0.0; querylat = 0.0;
allQueries = 0; allQueriesOK = 0;
for (i = 0; i < noWorkers; i++) {
ifstream resultFileIn(resultFile+StringFromVal(i));
if (!resultFileIn.is_open()) {
cerr << "cannot open results file to read\n";
exit(1);
}
int worker, queries_failed, total_queries;
double mspassed;
resultFileIn >> worker;
resultFileIn >> queries_failed;
resultFileIn >> total_queries;
resultFileIn >> mspassed;
resultFileIn.close();
allQueries += total_queries;
allQueriesOK += total_queries - queries_failed;
interval = max(interval, mspassed);//there should be just one worker reporting nonzero time
cerr << "parent: worker " << worker << ": mspased " << mspassed << " totalqueries " <<
total_queries << " queriesfailed " << queries_failed <<
" \n";
}
querytput = allQueriesOK*1000.0/interval;
querylat = interval * noWorkers/allQueries;
cerr <<"overall: throughput " << querytput << " queries/sec latency " << querylat << " msec/query \n";
char * ev = getenv("RUNEXP_LOG_RESPONSE");
if (ev != NULL) {
//let's log the response
ofstream f(ev);
f << querytput << "\n";
f << querylat << "\n";
f.close();
}
}
*/
static void
loadDB(const TestConfig & tc, std::string dbname, std::string dumpname) {
std::string comm = "mysql -u root -pletmein -e 'drop database if exists " + dbname + "; ' ";
std::cerr << comm << "\n";
assert_s(system(comm.c_str()) >= 0, "cannot drop db with if exists");
comm = "mysql -u root -pletmein -e 'create database " + dbname + ";' ";
std::cerr << comm << "\n";
assert_s(system(comm.c_str()) >= 0, "cannot create db");
if (dumpname != "") {
comm = "mysql -u root -pletmein " + dbname + " < " + tc.edbdir + "/../eval/dumps/" + dumpname + "; ";
std::cerr << comm << "\n";
assert_s(system(comm.c_str()) >= 0, "cannotload dump");
}
}
static void
startProxy(const TestConfig & tc, std::string host, uint port) {
setenv("CRYPTDB_LOG", cryptdb_logger::getConf().c_str(), 1);
setenv("CRYPTDB_MODE", "single", 1);
pid_t proxy_pid = fork();
if (proxy_pid == 0) {
LOG(test) << "starting proxy, pid " << getpid();
std::stringstream script_path, address, backend;
script_path << "--proxy-lua-script=" << tc.edbdir << "/../mysqlproxy/wrapper.lua";
address << "--proxy-address=localhost:" << port;
backend << "--proxy-backend-addresses=" << host << ":" << tc.port;
std::cerr << "starting proxy on port " << port << "\n";
std::cerr << "\n";
std::cerr << "mysql-proxy" << " --plugins=proxy" <<
" --max-open-files=1024 " <<
script_path.str().c_str() << " " <<
address.str().c_str() <<
backend.str().c_str() << "\n";
std::cerr << "\n";
execlp("mysql-proxy",
"mysql-proxy", "--plugins=proxy",
"--max-open-files=1024",
script_path.str().c_str(),
address.str().c_str(),
backend.str().c_str(),
(char *) 0);
LOG(warn) << "could not execlp: " << strerror(errno);
exit(-1);
} else if (proxy_pid < 0) {
assert_s(false,"failed to fork");
}
//back in parent, wait for proxy to start
std::cerr << "waiting for proxy to start\n";
sleep(1);
}
static void
testTrace(const TestConfig &tc, int argc, char ** argv)
{
std::cerr << "testTrace uses old EDBProxy -- not run" << std::endl;
/*
string masterKey = BytesFromInt(mkey, AES_KEY_BYTES);
//trace encrypt_db createsfile indexfile queriesfile insertsfile outputfile
EDBProxy * cl = NULL;
if (argc < 2) {
cerr << "usage: test trace encrypt_queries/eval\n";
return;
}
if (string(argv[1]) == "encrypt_queries") {
if (argc != 9) {
cerr << "trace encrypt_queries createsfile indexfile queriestotrainfile queriestotranslate baseoutputfile totallines"
" noWorkers \n";
return;
}
cl = new EDBProxy(tc.host, tc.user, tc.pass, tc.db, tc.port);
cl->setMasterKey(masterKey);
string queriestotranslate = argv[5];
string baseoutputfile = argv[6];
int totalLines = atoi(argv[7]);
int noWorkers = atoi(argv[8]);
dotrain(cl, argv[2], argv[4], "0");
assignWork(queriestotranslate, noWorkers, totalLines, 1, true);
pid_t pids[noWorkers];
for (int i = 0; i < noWorkers; i++) {
int index = i;
string work = workloads[index];
string outputfile = baseoutputfile + fixedRepr(index);
assert_s(system(("rm -f " + outputfile).c_str()) >= 0, "failed to remove " + outputfile);
pid_t pid = fork();
if (pid == 0) {
Stats * st = new Stats();
runQueriesFromFile(cl, work, false, true, NULL, outputfile, false, false, st, 100000);
cerr << "worker " << i << "finished\n";
exit(1);
} else if (pid < 0) {
cerr << "failed to fork \n";
exit(1);
} else { // in parent
pids[i] = pid;
}
}
int childstatus;
for (int i = 0; i < noWorkers; i++) {
if (waitpid(pids[i], &childstatus, 0) == -1) {
cerr << "there were problems with process " << pids[i]
<< "\n";
}
}
return;
};
if (string(argv[1]) == "eval") {
if (argc != 8) {
cerr << "trace eval queryfile totallines noworkers noRepeats split? logFreq \n";
return;
}
cl = new EDBProxy(tc.host, tc.user, tc.pass, tc.db, tc.port);
cl->setMasterKey(masterKey);
string queryfile = argv[2];
int totalLines = atoi(argv[3]);
int noWorkers = atoi(argv[4]);
int noRepeats = atoi(argv[5]);
bool split = atoi(argv[6]);
int logFreq = atoi(argv[7]);
assignWork(queryfile, noWorkers, totalLines, noRepeats, split);
runExp(cl, noWorkers, tc, logFreq);
delete cl;
return;
}
if (string(argv[1]) == "latency") {
if (argc < 5) {
cerr << "trace latency queryfile logFreq serverhost [optional: enctablesfile] \n";
return;
}
string queryfile = argv[2];
int logFreq = atoi(argv[3]);
string serverhost = argv[4];
string filename = "";
if (argc == 6) {
filename = argv[5];
}
//LOAD DUMP
if (serverhost == "localhost") {
loadDB(tc, "tpccenc", "up_dump_enc_w1");
}
//START PROXY
setenv("CRYPTDB_DB", "tpccenc", 1);
setenv("EDBDIR", tc.edbdir.c_str(), 1);
string tpccdir = tc.edbdir + "/../eval/tpcc/";
if (filename != "") {
setenv("LOAD_ENC_TABLES", filename.c_str(), 1);
}
setenv("TRAIN_QUERY", ("train 1 " + tpccdir +"sqlTableCreates " + tpccdir + "querypatterns_bench 0").c_str(), 1);
uint proxy_port = 5333;
int res = system("killall mysql-proxy;");
cerr << "killing proxy .. " << res << "\n";
sleep(2);
startProxy(tc, serverhost , proxy_port);
Connect * conn = new Connect(tc.host, tc.user, tc.pass, "tpccenc", proxy_port);
Stats * stats = new Stats();
runQueriesFromFile(NULL, queryfile, 1, 0, conn, "", false, false, stats, logFreq);
cerr << "latency " << stats->mspassed/stats->total_queries << "\n";
delete cl;
return;
}
*/
return;
}
static void
generateEncTables(const TestConfig & tc, int argc, char ** argv)
{
std::cerr << "generateEncTables uses old EDBProxy -- not run"
<< std::endl;
/*
if (argc!=2) {
cerr << "usage: gen_enc_tables filename\n";
return;
}
cerr << "generating enc tables for tpcc \n";
string filename = argv[1];
string masterKey = BytesFromInt(mkey, AES_KEY_BYTES);
EDBProxy * cl;
cl = new EDBProxy(tc.host, tc.user, tc.pass, tc.db, tc.port);
cl->setMasterKey(masterKey);
cl->execute("train 1 ../eval/tpcc/sqlTableCreates ../eval/tpcc/querypatterns_bench 0");
cerr << "a\n";
list<OPESpec> opes = {
{"new_order.no_o_id", 0, 11000},
{"oorder.o_id", 0, 11000},
{"order_line.ol_o_id", 0, 10000},
{"stock.s_quantity", 0, 100}
};
cerr << "b\n";
cl->generateEncTables(opes, 0, 10000, 20000, filename);
delete cl;
*/
}
static void
testEncTables(const TestConfig & tc, int argc, char ** argv) {
std::cerr << "testEncTables uses old EDBProxy -- not run"
<< std::endl;
/*if (argc!=3) {
cerr << "usage: test_enc_tables filename queriesfile\n";
return;
}
cerr << "loading enc tables for tpcc \n";
string filename = argv[1];
string queryfile = argv[2];
string masterKey = BytesFromInt(mkey, AES_KEY_BYTES);
EDBProxy * cl;
assert_s(system("mysql -u root -pletmein -e 'drop database if exists tpccenc;' ") >= 0, "cannot drop tpccenc with if exists");
assert_s(system("mysql -u root -pletmein -e 'create database tpccenc;' ") >= 0, "cannot create tpccenc");
cl = new EDBProxy(tc.host, tc.user, tc.pass, "tpccenc", tc.port);
cl->setMasterKey(masterKey);
cl->execute("train 1 ../eval/tpcc/sqlTableCreates ../eval/tpcc/querypatterns_bench 0");
cerr << "loading enc tables from file " << filename << "..";
cl->loadEncTables(filename);
cerr << "done! \n";
cerr << "load dump\n";
assert_s(system("mysql -u root -pletmein tpccenc < ../eval/dumps/up_dump_enc_w1") >=0, "cannot load dump");
Connect * conn = new Connect(tc.host, tc.user, tc.pass, "tpccenc", tc.port);
Stats * stats = new Stats();
runQueriesFromFile(cl, queryfile, 1, 1, conn, "", 0, false, stats, 1000);
*/
}
/*
cerr << "trying to execute " << (path+"loadData.sh") << " " << "mysqlproxy.properties" << "\n";
execlp((path+"loadData.sh").c_str(),
"loadData.sh", "mysqlproxy.properties",
"numWarehouses",
"4",
(char *) 0
);
LOG(warn) << "could not execlp bench: " << strerror(errno);
*/
static void
testBench(const TestConfig & tc, int argc, char ** argv)
{
if (argc < 2) {
std::cerr << "usage: bench logplain or [client,server,both]";
return;
}
if (std::string(argv[1]) == "logplain") {
if (argc != 3){
std::cerr << "usage: bench logplain nowarehouses;";
return;
}
std::string numWarehouses = argv[2];
loadDB(tc, "cryptdbtest", "");
std::string comm = "mysql -u root -pletmein cryptdbtest < ../../tpcc/orig_table_creates ;";
std::cerr << comm << "\n";
assert_s(system(comm.c_str()) >= 0, "cannot create tables");
setenv("CRYPTDB_MODE", "single", 1);
setenv("DO_CRYPT", "false", 1);
//setenv("EXECUTE_QUERIES", "false", 1);
setenv("LOG_PLAIN_QUERIES", (std::string("plain_insert_w")+numWarehouses).c_str(), 1);
setenv("CRYPTDB_DB", "cryptdbtest", 1);
setenv("EDBDIR", tc.edbdir.c_str(), 1);
setenv("CRYPTDB_LOG", cryptdb_logger::getConf().c_str(), 1);
std::string tpccdir = tc.edbdir + "/../eval/tpcc/";
int res = system("killall mysql-proxy;");
sleep(2);
std::cerr << "killing proxies .. " <<res << "\n";
//start proxy(ies)
startProxy(tc, "127.0.0.1", 5133);
comm = std::string("java -cp ../build/classes:../lib/edb-jdbc14-8_0_3_14.jar:../lib/ganymed-ssh2-build250.jar:") +
"../lib/hsqldb.jar:../lib/mysql-connector-java-5.1.10-bin.jar:../lib/ojdbc14-10.2.jar:../lib/postgresql-8.0.309.jdbc3.jar " +
"-Ddriver=com.mysql.jdbc.Driver " +
"-Dconn=jdbc:mysql://localhost:5133/cryptdbtest " +
"-Duser=root -Dpassword=letmein LoadData.LoadData numWarehouses " + numWarehouses;
std::cerr << "\n" << comm << "\n\n";
assert_s(system(comm.c_str())>=0, "problem running benchmark");
return;
}
if((argc != 3) && (argc != 9)) {
std::cerr << "usage: bench role[client, server, both] encrypted?[1,0] [specify for client: proxyhost serverhost noWorkers oneProxyPerWorker[1/0] noWarehouses timeLimit(mins)] \n";
exit(-1);
}
std::string role = argv[1];
bool encrypted = atoi(argv[2]);
bool is_client = false;
bool is_server = false;
uint baseport = 5133;
if (role == "both") {
is_client = true;
is_server = true;
} else {
if (role == "server") {
is_server = true;
} else
{
if (role != "client") {
std::cerr <<"invalid role\n";
} else {
is_client = true;
}
}
}
std::string serverhost = "localhost", proxyhost = "localhost", timeLimit="", noWarehouses="";
bool oneProxyPerWorker = 0;
unsigned int noWorkers = 0;
if (is_client) {
proxyhost = argv[3];
serverhost = argv[4];
noWorkers = atoi(argv[5]);
oneProxyPerWorker = atoi(argv[6]);
assert_s(oneProxyPerWorker == 0 || oneProxyPerWorker == 1, "oneProxyPerWorker should be 0 or 1");
noWarehouses = argv[7];
timeLimit = argv[8];
}
if (encrypted) {
if (is_server) {
loadDB(tc, "tpccenc", "up_dump_enc_w1");
}
if (is_client) {
setenv("EDBDIR", tc.edbdir.c_str(), 1);
setenv("CRYPTDB_LOG", cryptdb_logger::getConf().c_str(), 1);
setenv("CRYPTDB_MODE", "single", 1);
setenv("CRYPTDB_DB", "tpccenc", 1);
std::string tpccdir = tc.edbdir + "/../eval/tpcc/";
setenv("LOAD_ENC_TABLES", (tpccdir+"enc_tables_w1").c_str(), 1);
//configure proxy
setenv("TRAIN_QUERY", ("train 1 " + tpccdir +"sqlTableCreates " + tpccdir + "querypatterns_bench 0").c_str(), 1);
int res = system("killall mysql-proxy;");
sleep(2);
std::cerr << "killing proxies .. " <<res << "\n";
//start proxy(ies)
if (!oneProxyPerWorker) {
startProxy(tc, serverhost, baseport);
} else {
for (unsigned int i = 0 ; i < noWorkers; i++) {
startProxy(tc, serverhost, baseport + i);
}
}
std::string comm = std::string("java") + " -cp ../build/classes:../lib/edb-jdbc14-8_0_3_14.jar:../lib/ganymed-ssh2-build250.jar:" +
"../lib/hsqldb.jar:../lib/mysql-connector-java-5.1.10-bin.jar:../lib/ojdbc14-10.2.jar:../lib/postgresql-8.0.309.jdbc3.jar " +
"-Ddriver=com.mysql.jdbc.Driver " +
"-Dconn=jdbc:mysql://"+proxyhost+":"+StringFromVal(baseport)+"/tpccenc " +
"-Duser=root -Dpassword=letmein " +
"-Dnwarehouses="+noWarehouses+" -Dnterminals=" + StringFromVal(noWorkers)+
" -DtimeLimit="+timeLimit+" client.jTPCCHeadless";
std::cerr << "\n" << comm << "\n\n";
assert_s(system(comm.c_str())>=0, "problem running benchmark");
}
} else {
if (is_server) {
loadDB(tc, "tpccplain", "dump_plain_w8");
}
//just start the benchmark
if (is_client) {
assert_s(system((std::string("java") + " -cp ../build/classes:../lib/edb-jdbc14-8_0_3_14.jar:../lib/ganymed-ssh2-build250.jar:"
"../lib/hsqldb.jar:../lib/mysql-connector-java-5.1.10-bin.jar:../lib/ojdbc14-10.2.jar:../lib/postgresql-8.0.309.jdbc3.jar "
"-Ddriver=com.mysql.jdbc.Driver "
"-Dconn=jdbc:mysql://"+serverhost+":3306/tpccplain "
"-Duser=root -Dpassword=letmein "
"-Dnwarehouses="+noWarehouses+" -Dnterminals=" + StringFromVal(noWorkers) +
" -DtimeLimit="+timeLimit+" client.jTPCCHeadless").c_str())>=0,
"problem running benchmark");
}
}
//don't forget to create indexes!
}
/*
bool outputOnions = false;
if (argc == 6) {
outputOnions = argv[4];
}
*/
// cl->plain_execute("DROP TABLE IF EXISTS
// phpbb_acl_groups,phpbb_acl_options,phpbb_acl_roles,
// phpbb_acl_roles_data,phpbb_acl_users,phpbb_attachments,
// phpbb_banlist,phpbb_bbcodes,phpbb_bookmarks,phpbb_bots,
// phpbb_config,phpbb_confirm,phpbb_disallow,phpbb_drafts,
// phpbb_extension_groups,phpbb_extensions,phpbb_forums,
// phpbb_forums_access,phpbb_forums_track,phpbb_forums_watch,
// phpbb_groups,phpbb_icons,phpbb_lang,phpbb_log,phpbb_moderator_cache,
// phpbb_modules,phpbb_poll_options,phpbb_poll_votes,phpbb_posts,
// phpbb_privmsgs,phpbb_privmsgs_folder,phpbb_privmsgs_rules,
// phpbb_privmsgs_to,phpbb_profile_fields,phpbb_profile_fields_data,
// phpbb_profile_fields_lang,phpbb_profile_lang,phpbb_ranks,
// phpbb_reports,phpbb_reports_reasons,phpbb_search_results,
// phpbb_search_wordlist,phpbb_search_wordmatch,phpbb_sessions,
// phpbb_sessions_keys,phpbb_sitelist,phpbb_smilies,phpbb_styles,
// phpbb_styles_imageset,phpbb_styles_imageset_data,
// phpbb_styles_template,phpbb_styles_template_data,phpbb_styles_theme,
// phpbb_topics,phpbb_topics_posted,phpbb_topics_track,
// phpbb_topics_watch,phpbb_user_group,phpbb_users,
// phpbb_warnings,phpbb_words,phpbb_zebra;");
/*
ifstream createsfile(argv[2]);
ifstream tracefile(argv[3]);
string query;
int noinstr = atoi(argv[4]);
if (!createsfile.is_open()) {
cerr << "cannot open " << argv[2] << "\n";
exit(1);
}
while (!createsfile.eof()) {
getline(createsfile, query);
cerr << "line is < " << query << ">\n";
if (query.length() < 3) { continue; }
//list<string> q = cl->rewriteEncryptQuery(query, rb);
//assert_s(q.size() == 1, "query translated has more than one query or
// no queries;");
list<string> res = cl->rewriteEncryptQuery(query);
//cerr << "problem with query!\n";
assert_s(res.size() == 1, "query did not return one");
//cout << q.front() << "\n";
}
cerr << "creates ended \n";
if (!tracefile.is_open()) {
cerr << "cannot open " << argv[3] << "\n";
exit(1);
}
struct timeval starttime, endtime;
gettimeofday(&starttime, NULL);
for (int i = 0; i <noinstr; i++) {
if (!tracefile.eof()) {
getline(tracefile, query);
//if ((i<1063000) && (i>17)) {continue;}
//cerr << "line is < " << query << ">\n";
if (i % 100 == 0) {cerr << i << "\n"; }
if (query.length() < 3) { continue; }
//list<string> q = cl->rewriteEncryptQuery(query,
// rb);
//assert_s(q.size() == 1, "query translated has more than one
// query or no queries;");
if (!cl->execute(query).ok) {
cerr << "problem with query!\n";
}
//cout << q.front() << "\n";
} else {
cerr << "instructions ended \n";
break;
}
}
gettimeofday(&endtime, NULL);
cout << (noinstr*1.0/timeInSec(starttime,endtime)) << "\n";
//cerr << "DONE with trace \n";
if (outputOnions) {
cerr << "outputting state of onions ... \n";
cl->outputOnionState();
}
tracefile.close();
*/
static void
test_PKCS(const TestConfig &tc, int ac, char **av)
{
PKCS * pk,* sk;
generateKeys(pk, sk);
assert_s(pk != NULL, "pk is null");
assert_s(sk != NULL, "pk is null");
std::string pkbytes = marshallKey(pk, true);
assert_s(pkbytes ==
marshallKey(unmarshallKey(pkbytes,
1),
1), "marshall does not work");
std::string skbytes = marshallKey(sk, false);
assert_s(skbytes ==
marshallKey(unmarshallKey(skbytes,
0),
0), "marshall does not work");
char msg[] = "Hello world";
std::string enc = encrypt(pk, msg);
std::string dec = decrypt(sk, enc);
assert_s(msg == dec, "decryption is not original msg");
std::cerr << "msg" << dec << "\n";
}
static void help(const TestConfig &tc, int ac, char **av);
static struct {
const char *name;
const char * description;
void (*f)(const TestConfig &, int ac, char **av);
} tests[] = {
//{ "aes", "", &evaluate_AES },
{ "autoinc", "", &autoIncTest },
//{ "consider", "consider queries (or not)", &TestNotConsider::run },
//{ "crypto", "crypto functions", &TestCrypto::run },
//{ "paillier", "", &testPaillier },
{ "parseaccess", "", &testParseAccess },
{ "pkcs", "", &test_PKCS },
//{ "proxy", "proxy", &TestProxy::run },
{ "queries", "queries", &TestQueries::run },
//{ "single", "integration - single principal",&TestSinglePrinc::run },
{ "gen_enc_tables", "", &generateEncTables },
{ "test_enc_tables","", &testEncTables },
{ "trace", "trace eval", &testTrace },
{ "bench", "TPC-C benchmark eval", &testBench },
//{ "utils", "", &testUtils },
{ "train", "", &testTrain },
{ "help", "", &help },
};
static void
help(const TestConfig &tc, int ac, char **av)
{
std::cerr << "Usage: " << av[0] << " [options] testname" << std::endl;
std::cerr << "Options:" << std::endl
<< " -s stop on failure [" << tc.stop_if_fail << "]" << std::endl
<< " -h host database server [" << tc.host << "]" << std::endl
<< " -t port database port [" << tc.port << "]" << std::endl
<< " -u user database username [" << tc.user << "]" << std::endl
<< " -p pass database password [" << tc.pass << "]" << std::endl
<< " -d db database to use [" << tc.db << "]" << std::endl
<< " -e db embedded database to use [" << tc.shadowdb_dir << "]" << std::endl
<< " -v group enable verbose messages in group" << std::endl;
std::cerr << "Verbose groups:" << std::endl;
for (auto i = log_name_to_group.begin(); i != log_name_to_group.end(); i++)
std::cerr << " " << i->first << std::endl;
std::cerr << "Supported tests:" << std::endl;
for (uint i = 0; i < NELEM(tests); i++)
std::cerr << " " << std::left << std::setw(20)
<< tests[i].name << tests[i].description << std::endl;
}
int
main(int argc, char ** argv)
{
TestConfig tc;
int c;
while ((c = getopt(argc, argv, "v:sh:u:p:d:t:e:")) != -1) {
switch (c) {
case 'v':
if (log_name_to_group.find(optarg) == log_name_to_group.end()) {
help(tc, argc, argv);
exit(0);
}
cryptdb_logger::enable(log_name_to_group[optarg]);
break;
case 's':
tc.stop_if_fail = true;
break;
case 'u':
tc.user = optarg;
break;
case 'p':
tc.pass = optarg;
break;
case 'd':
tc.db = optarg;
break;
case 'h':
tc.host = optarg;
break;
case 't':
tc.port = atoi(optarg);
break;
case 'e':
tc.shadowdb_dir = optarg;
break;
default:
help(tc, argc, argv);
exit(0);
}
}
if (argc == optind) {
std::cerr << "interactiveTest is deprecated; please use obj/parser/cdb_test" << std::endl;
return 0;
}
for (uint i = 0; i < NELEM(tests); i++) {
if (!strcasecmp(argv[optind], tests[i].name)) {
tests[i].f(tc, argc - optind, argv + optind);
// TODO: Check if there are mem leaks before exiting.
return 0;
}
}
help(tc, argc, argv);
}
/* if (strcmp(argv[1], "train") == 0) {
test_train();
return 0;
}*/
/* if (strcmp(argv[1], "trace") == 0) {
if (argc != 4) { cerr << "usage ./test trace file
noqueries isSecure ";}
runTrace(argv[2], atoi(argv[3]), atoi(argv[4]));
return 0;
} */ /*
if (strcmp(argv[1], "convert") == 0) {
convertQueries();
return 0;
}
if (strcmp(argv[1], "instance") == 0) {
createInstance();
return 0;
}
*/
/*
if (strcmp(argv[1], "convertdump") == 0) {
convertDump();
return 0;
}
*/
/* if (strcmp(argv[1], "load") == 0) {
if (argc != 8) {
cerr << "usage: test load noWorkers totalLines logFreq
file workeri1 workeri2\n";
exit(1);
}
string filein = argv[5];
cerr << "input file is " << argv[5] << "\n";
parallelLoad(atoi(argv[2]), atoi(argv[3]), atoi(argv[4]),
filein, atoi(argv[6]), atoi(argv[7]));
return 0;
}
//int noWorkers, int noRepeats, int logFreq, int totalLines, string
dfile, bool hasTransac
if (strcmp(argv[1], "throughput") == 0) {
if (argc != 9) {
cerr << "usage: test throughput noWorkers noRepeats
logFreq totalLines queryFile isSecure hasTransac
\n";
exit(1);
}
simpleThroughput(atoi(argv[2]), atoi(argv[3]), atoi(argv[4]),
atoi(argv[5]), string(argv[6]), atoi(argv[7]),
atoi(argv[8]));
}
if (strcmp(argv[1], "latency") == 0) {
// queryFile, maxQueries, logFreq, isSecure, isVerbose
if (argc != 7) {
cerr << "usage: test latency queryFile maxQue
logFreq isSecure isVerbose \n";
exit(1);
}
latency(string(argv[2]), atoi(argv[3]), atoi(argv[4]),
atoi(argv[5]), atoi(argv[6]));
}
*/ /*
if (strcmp(argv[1], "integration") == 0){
testEDBProxy();
return 0;
}
cerr << "unknown option\n";
//testCryptoManager();
//testEDBProxy();
//tester t = tester(tc, randomBytes(AES_KEY_BYTES));
//t.testClientParser();
//tester t = tester(tc);
//t.testMarshallBinary();
//microEvaluate(argc, argv); //microEvaluate
//test_OPE();
//test_HGD();
//test_EDBProxy_noSecurity();
//evaluateMetrics(argc, argv);
*/
