diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c6e1c91 --- /dev/null +++ b/.gitignore @@ -0,0 +1,20 @@ +*~ +*.o +COPYING +INSTALL +Makefile.in +aclocal.m4 +autom4te.cache +autoscan-2.68.log +compile +config.* +configure +depcomp +install-sh +missing +Makefile +src/.deps +src/qpi_stress +stamp-h1 +cscope* + diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..7810370 --- /dev/null +++ b/AUTHORS @@ -0,0 +1 @@ +fangdingjun diff --git a/ChangeLog b/ChangeLog new file mode 100644 index 0000000..2a7c725 --- /dev/null +++ b/ChangeLog @@ -0,0 +1,3 @@ + +2011-7-16: +initial project diff --git a/Makefile.am b/Makefile.am new file mode 100644 index 0000000..af437a6 --- /dev/null +++ b/Makefile.am @@ -0,0 +1 @@ +SUBDIRS = src diff --git a/NEWS b/NEWS new file mode 100644 index 0000000..5a1387d --- /dev/null +++ b/NEWS @@ -0,0 +1,3 @@ + +2011-7-16 +initial projects diff --git a/README b/README new file mode 100644 index 0000000..4fcb45c --- /dev/null +++ b/README @@ -0,0 +1,30 @@ + +1.overview + + this is a tool to test CPU's QPI channel: + + start a process A and bind to cpu0, then request a share memory + start a process B and bind to cpu1, then attach to process A's share memory + process A write to the share memory, process B read from then share memory via QPI channel + process A read the process B's share memory via QPI channel + + procA procB + || || + +------+ +--------+ + |cpu0 | QPI channel | cpu1 | + +------+ +--------+ + || || + +-----+ +-----+ + |memA | |memB | + +-----+ +-----+ + +2. install + unpack the source and enter the source directory, type + ./configure + make + make install + +3.usage + type + qpi_stress -h + for help diff --git a/autogen.sh b/autogen.sh new file mode 100755 index 0000000..7db7e5c --- /dev/null +++ b/autogen.sh @@ -0,0 +1,2 @@ +#!/bin/sh +aclocal && autoheader && autoconf && automake -a && ./configure diff --git a/configure.ac b/configure.ac new file mode 100644 index 0000000..3f9743c --- /dev/null +++ b/configure.ac @@ -0,0 +1,28 @@ +# -*- Autoconf -*- +# Process this file with autoconf to produce a configure script. + +AC_PREREQ([2.68]) +AC_INIT([qpi-stress], [0.1], [fangdingjun@gmail.com]) +AC_CONFIG_SRCDIR([src/worker.c]) +AC_CONFIG_HEADERS([config.h]) +AM_INIT_AUTOMAKE(1.0) + +# Checks for programs. +AC_PROG_CC +AM_PROG_CC_C_O + +# Checks for libraries. + +# Checks for header files. +AC_CHECK_HEADERS([fcntl.h stdlib.h string.h sys/time.h unistd.h]) + +# Checks for typedefs, structures, and compiler characteristics. +AC_TYPE_PID_T +AC_TYPE_SIZE_T + +# Checks for library functions. +AC_FUNC_FORK +AC_CHECK_FUNCS([alarm gettimeofday memset]) +AC_CONFIG_FILES([Makefile + src/Makefile]) +AC_OUTPUT diff --git a/src/Makefile.am b/src/Makefile.am new file mode 100644 index 0000000..6647d72 --- /dev/null +++ b/src/Makefile.am @@ -0,0 +1,4 @@ +bin_PROGRAMS = qpi_stress +qpi_stress_SOURCES = main.c cpu.c shm.c report.c worker.c +qpi_stress_CFLAGS = -Wall -Werror -D_GNU_SOURCE +qpi_stress_LDFLAGS = -lpthread diff --git a/src/cpu.c b/src/cpu.c new file mode 100644 index 0000000..bb4f738 --- /dev/null +++ b/src/cpu.c @@ -0,0 +1,47 @@ +/* + * Filename: cpu.c + * Author: fangdingjun@gmail.com + * License: GPLv2 + * Date: 2011-7-10 + * + */ + +#include +#include +#include +//#include "qpi.h" + +int bind_to_cpu(int cpunum) +{ + int num = sysconf(_SC_NPROCESSORS_CONF); + int i; + + cpu_set_t mask; + + CPU_ZERO(&mask); + CPU_SET(cpunum, &mask); + + if (sched_setaffinity(0, sizeof(mask), &mask) == -1) { + perror("set cpu affinity failed"); + return -1; + } + + CPU_ZERO(&mask); + if (sched_getaffinity(0, sizeof(mask), &mask) == -1) { + perror("get cpu affinity failed"); + return -1; + } + for (i = 0; i < num; i++) { + if (CPU_ISSET(i, &mask)) { + break; + } + } + if (i != cpunum) { + fprintf(stderr, + "set cpu affinity failed: process not running on cpu %d\n", + cpunum); + return -1; + } + return 0; +} + diff --git a/src/main.c b/src/main.c new file mode 100644 index 0000000..5659c4b --- /dev/null +++ b/src/main.c @@ -0,0 +1,302 @@ +/* + * this is tool to stress cpu's qpi channel on romely platform + * + * Filename: main.c + * Author: fangdingjun@gmail.com + * License: GPLv2 + * Date: 2011-7-10 + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "qpi.h" + + +int main(int argc, char **argv) +{ + /* initial global value */ + proc_num = 2; + data = 0xaa; + shm_size = 4 << 20; // 4 MB + running = 1; + output = NULL; + fp = NULL; + ret = 0; + + int opt_idx; + unsigned int timeout = 0; + int delay = 10; + long page_size; + long total_page; + unsigned long req_mem; + unsigned long avb_mem; + int c; + int i; + key_t k_read; + key_t k_write; + key_t k_error; + //int id_r,id_w; + + struct option opt[] = { + {"timeout", 1, 0, 't'}, + {"number", 1, 0, 'n'}, + {"data", 1, 0, 'd'}, + {"size", 1, 0, 's'}, + {"help", 0, 0, 'h'}, + {"delay", 1, 0, 'p'}, + {"output", 1, 0, 'o'}, + {0, 0, 0, 0} + }; + proc_num = cpu_num = sysconf(_SC_NPROCESSORS_CONF); + // process command line options + while (1) { + c = getopt_long(argc, argv, "hn:t:d:s:p:o:", opt, &opt_idx); + if (c == -1) { + break; + } + switch (c) { + case 'n': + proc_num = atoi(optarg); + break; + case 't': + timeout = atoi(optarg) * 60; + break; + case 'd': + //data=atoi(optarg); + if (!memcmp("0x", optarg, 2)) { + sscanf(optarg, "%x", &data); + } else { + data = atoi(optarg); + } + break; + case 's': + shm_size = atoi(optarg) << 20; + break; + case 'p': + delay = atoi(optarg); + break; + case 'o': + output = optarg; + break; + case 'h': + usage(); + break; + default: + usage(); + } + } + if (argc > optind) { + usage(); + } + // process number error + page_size = sysconf(_SC_PAGESIZE); + total_page = sysconf(_SC_PHYS_PAGES); + req_mem = proc_num * shm_size; + avb_mem = page_size * total_page; + //fp=stderr; + if (output != NULL) { + fp = fopen(output, "w"); + if (fp == NULL) { + fprintf(stderr, + "WARNING: open %s failed, log to screen\n", + output); + //fp=stderr; + } + } + // 0 for random data + if (data == 0) { + srand(time(NULL)); + data = rand() % 0xff; + } + //printf("argc %d optind %d\n",argc,optind); + //return 0; + fprintf(stdout, "summay:\n" + "\ttotal processors: %ld\n" + "\ttotal memory: %ld MB\n" + "\tstart processes: %d\n" + "\tdata partern: 0x%02x\n" + "\truntime: %u seconds\n" + "\tresult display delay: %d seconds\n" + "\tper process share memory size: %lu MB\n\n", + cpu_num, + avb_mem >> 20, + proc_num, data, + timeout == + 0 ? (unsigned int)-1 : (unsigned int)timeout, delay, + shm_size >> 20); + + if (proc_num % 2) { + fprintf(stderr, + "ERROR: process number must be 2, 4, 6, 8, ...\n"); + exit(-1); + } + + if (proc_num > cpu_num) { + fprintf(stderr, + "ERROR: only %ld processors core," + " but you want to run %d processes\n", + cpu_num, proc_num); + exit(-1); + } + + if (req_mem > avb_mem) { + fprintf(stderr, + "ERROR: total available memory is %ld bytes," + " but you request %d * %lu = %lu bytes\n", + avb_mem, proc_num, shm_size, req_mem); + exit(-1); + } + + sem_t *l_sem[proc_num]; + sem = l_sem; + key_t a_key[proc_num]; + key = a_key; + init_shm_key(key, proc_num); + init_sem(sem, proc_num); + init_one_key(&k_read, "/dev/shm/read_counter"); + init_one_key(&k_write, "/dev/shm/write_counter"); + init_one_key(&k_error, "/dev/shm/error_counter"); + total_r = (int *)create_shm(k_read, sizeof(int)); + total_w = (int *)create_shm(k_write, sizeof(int)); + total_e = (int *)create_shm(k_error, sizeof(int)); + if (!(total_r && total_w && total_e)) { + //exit(-1); + ret = -1; + goto out; + } + *total_e = 0; + *total_r = 0; + *total_w = 0; + + read_lock = sem_open("read", O_CREAT, 0644, 0); + if (read_lock == SEM_FAILED) { + perror("sem_open"); + //exit(-1); + ret = -1; + goto out; + } + sem_init(read_lock, 1, 0); + + write_lock = sem_open("write", O_CREAT, 0644, 0); + if (write_lock == SEM_FAILED) { + perror("sem_open"); + //exit(-1); + ret = -1; + goto out; + } + sem_init(write_lock, 1, 0); + + for (i = 0; i < proc_num; i++) { + if (do_fork(i) < 0) { // fork subprocess + ret = -1; + goto out1; + } + } + + usleep(100); + fprintf(stdout, "-------------------------------\n"); + fprintf(stdout, "\nQPI stress staring ...\n\n"); + //sleep(1); + + signal(SIGINT, sig); + signal(SIGTERM, sig); + signal(SIGHUP, sig); + signal(SIGALRM, sig); + signal(SIGUSR1, sig); + signal(SIGUSR2, sig); + + // register timer + if (timeout) { + alarm(timeout); + } + //report status + report(delay); + +out1: + kill(0, SIGUSR2); // terminate children process + sleep(2); + + // check if data check error + if (*total_e) { + fprintf(stderr, "data check error: %d\n", *total_e); + ret = -1; + } +out: + // destroy share memory and semaphore + destroy_shm(key, proc_num); + destroy_sem(sem, proc_num); + + /* cleaning semaphore */ + sem_close(write_lock); + sem_close(read_lock); + sem_unlink("write"); + sem_unlink("read"); + + delete_shm_by_name("/dev/shm/read_counter", sizeof(int)); + delete_shm_by_name("/dev/shm/write_counter", sizeof(int)); + delete_shm_by_name("/dev/shm/error_counter", sizeof(int)); + + if(fp){ + fclose(fp); + } + return ret; +} + +int delete_shm_by_name(char *n, size_t s) +{ + key_t k; + int id; + k = ftok(n, 0); + id = shmget(k, s, 0644); + if (id != -1) { + shmctl(id, IPC_RMID, NULL); + } + unlink(n); + return 0; +} + +void usage() +{ + fprintf(stderr, "Usage: " + "qpi_stress [options]" + "\n\t-h, --help\n\t\tprint this help" + "\n\t-n, --number=n\n\t\t start n processes, default number of cpu cores\n" + "\t-t, --timeout=n\n\t\t time to run(minute), default 0-run util CTRL+C pressed\n" + "\t-d, --data=n\n\t\t data to fill memory, 0 for random, default 0xaa\n" + "\t-s, --size=n\n\t\t size of share memory(MB), default 4\n" + "\t-p, --delay=n\n\t\t result display delay(seconds), default 10\n" + "\t-o, --output=file\n\t\t write result to file, default screen\n" + "Example:\n\t" + "qpi_stress -n4 -s10 -t10 -d 0x55" + "\n start 4 processes, run 10 minutes, per process requst 10 MB memory, use 0x55 to fill\n"); + exit(-1); +} + +void sig(int num) +{ + //printf("get signal %d, cleaning and exit ...\n",num); + if (num == SIGALRM) { + fprintf(stderr, "timeout, cleaning and exiting...\n"); + } else if (num == SIGINT) { + fprintf(stderr, "CTRL+C pressed, aborting...\n"); + } else if (num == SIGUSR1) { + fprintf(stderr, "error occured, aborting...\n"); + ret = 1; + } else if (num == SIGUSR2) { + ; + } else { + fprintf(stderr, "got signal %d, exiting...\n", num); + } + running = 0; + //return; +} diff --git a/src/qpi.h b/src/qpi.h new file mode 100644 index 0000000..66adc8b --- /dev/null +++ b/src/qpi.h @@ -0,0 +1,33 @@ +#ifndef QPI_H +#define QPI_H + sem_t *read_lock; + sem_t *write_lock; + int proc_num; + unsigned int data; + long cpu_num; + int *total_r; + int *total_w; + int *total_e; + key_t *key; + sem_t **sem; + unsigned long shm_size; + volatile int running; + char *output; + FILE *fp; + void usage(); + int init_shm_key(key_t * k, int num); + int init_sem(sem_t ** s, int num); + int init_one_key(key_t * k, char *name); + int do_fork(int m); + int worker(int num); + int destroy_shm(key_t * k, int num); + int destroy_sem(sem_t ** s, int num); + int report(int delay); + int delete_shm_by_name(char *n, size_t s); + int ret; + void sig(int num); + void *create_shm(key_t k, size_t s); + int bind_to_cpu(int cpunum); + void *create_shm(key_t k, size_t s); + void *wait_remote_shm(key_t k, size_t s); +#endif diff --git a/src/report.c b/src/report.c new file mode 100644 index 0000000..e396ce9 --- /dev/null +++ b/src/report.c @@ -0,0 +1,114 @@ +/* + * Filename: report.c + * Author: fangdingjun@gmail.com + * License: GPLv2 + * Date: 2011-7-10 + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include "qpi.h" + +#define DIFF(start,end) ((end.tv_sec-start.tv_sec) * 1000 + (end.tv_usec-start.tv_usec)) + +int report(int delay) +{ + struct timeval tv, tv1; + unsigned long tmp_read = 0, tmp_write = 0; + unsigned long timedelay = 0; + unsigned long mem_total_r = 0, mem_total_w = 0; + time_t now; + struct tm *tt; + char t_out[100]; + int i; + + + if (fp){ + fprintf(fp, "0 Read_perf_MB/s Read_delay_ms " + "Write_perf_MB/s Write_delay_ms\n"); + } + + // wake up thread + for (i = 0; i < proc_num; i++) { + //printf("sem_post(&shm_sem[%d])\n",i); + sem_post(sem[i]); + } + + sem_post(read_lock); + sem_post(write_lock); + gettimeofday(&tv, NULL); + while (running) { + memcpy(&tv1, &tv, sizeof(struct timeval)); + gettimeofday(&tv, NULL); + + timedelay = DIFF(tv1, tv); + sem_wait(read_lock); + tmp_read = *total_r; + *total_r = 0; + sem_post(read_lock); + + sem_wait(write_lock); + tmp_write = *total_w; + *total_w = 0; + sem_post(write_lock); + + //printf("timedelay %lu, read = %lu,write=%lu\n",timedelay,tmp_read,tmp_write); + + now = time(NULL); + tt = localtime(&now); + if (tmp_read && tmp_write && timedelay) { + mem_total_r = (tmp_read * shm_size) >> 20; + mem_total_w = (tmp_write * shm_size) >> 20; + //printf("%s, read %lu MB," " write %lu MB," + // " per %d seconds\n", + //ctime(&now), + // t_out, mem_total_r, mem_total_w, delay); + strftime(t_out, sizeof(t_out), "%Y-%m-%d %H:%M:%S", tt); + if (!fp) { + fprintf(stdout, "in recent %lu r/w(s):\n" + "%s\n" + "read: %4.2f MB/s, delay: %4.2f ms, write: %4.2f MB/s, delay: %4.2f ms\n\n", + tmp_read + tmp_write, + t_out, + (float)mem_total_r * 1000 / + (float)timedelay, + (float)timedelay / (float)tmp_read, + (float)mem_total_w * 1000 / + (float)timedelay, + (float)timedelay / (float)tmp_write); + + } else { + strftime(t_out, sizeof(t_out), "%m_%d_%H:%M:%S", + tt); + fprintf(fp, "%s %4.2f %4.2f %4.2f %4.2f\n", + t_out, + (float)mem_total_r * 1000 / + (float)timedelay, + (float)timedelay / (float)tmp_read, + (float)mem_total_w * 1000 / + (float)timedelay, + (float)timedelay / (float)tmp_write); + } + } + //total_old_r = total_r; + //total_old_w = total_w; + // if error, stop it + if (*total_e) { + //printf("data check error: %d\n",error); + kill(0, SIGUSR1); + break; + } + //printf("total=%d\n",total); + //gettimeofday(&tv1,NULL); + sleep(delay); + } + //fclose(fp); + return 0; +} + diff --git a/src/shm.c b/src/shm.c new file mode 100644 index 0000000..2f3c1c3 --- /dev/null +++ b/src/shm.c @@ -0,0 +1,131 @@ +/* + * Filename: report.c + * Author: fangdingjun@gmail.com + * License: GPLv2 + * Date: 2011-7-10 + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include "qpi.h" + +void *create_shm(key_t k, size_t s) +{ + int id; + void *p; + id = shmget(k, s, IPC_CREAT | 0644); + if (id == -1) { + perror("shmget"); + return NULL; + } + p = shmat(id, NULL, 0); + if (p == (void *)-1) { + perror("shmat"); + return NULL; + } + return p; +} + +int init_one_key(key_t * k, char *name) +{ + FILE *fp = fopen(name, "w"); + fclose(fp); + *k = ftok(name, 0); + return 0; +} + +int init_shm_key(key_t * k, int num) +{ + int i; + char fn_r[] = "/dev/shm/qpitestxxxxx"; + char fname[256]; + for (i = 0; i < num; i++) { + sprintf(fname, "%s%d", fn_r, i); + FILE *fp = fopen(fname, "w"); + fclose(fp); + k[i] = ftok(fname, 0); + } + return 0; +} + +int destroy_shm(key_t * k, int num) +{ + int i; + int id; + char fn_r[] = "/dev/shm/qpitestxxxxx"; + char fname[256]; + for (i = 0; i < num; i++) { + id = shmget(k[i], shm_size, 0644); + if (id != -1) { + shmctl(id, IPC_RMID, NULL); + } + sprintf(fname, "%s%d", fn_r, i); + unlink(fname); + } + return 0; +} + +int destroy_sem(sem_t ** s, int num) +{ + int i; + char sem_r[] = "semxxxx"; + char sem_n[256]; + for (i = 0; i < num; i++) { + sem_close(s[i]); + sprintf(sem_n, "%s%d", sem_r, i); + sem_unlink(sem_n); + } + return 0; +} + +int init_sem(sem_t ** s, int num) +{ + int i; + char sem_r[] = "semxxxx"; + char sem_n[256]; + for (i = 0; i < num; i++) { + sprintf(sem_n, "%s%d", sem_r, i); + s[i] = sem_open(sem_n, O_CREAT, 0644, 0); + if (s[i] == SEM_FAILED) { + perror("sem_open"); + exit(-1); + } + sem_init(s[i], 1, 0); + } + return 0; +} + +void *wait_remote_shm(key_t k, size_t s) +{ + void *p; + int id; + while (1) { // wait remote to initial share memory + id = shmget(k, s, 0644); + if (id == -1) { + if (errno == ENOENT) { + usleep(10); + } else { + perror("shmget"); + //exit(-1); + return NULL; + } + } else { + break; + } + } + + p = shmat(id, NULL, 0); + if (p == (void *)-1) { + perror("shmat"); + //exit(-1); + return NULL; + } + return p; +} + diff --git a/src/worker.c b/src/worker.c new file mode 100644 index 0000000..355813b --- /dev/null +++ b/src/worker.c @@ -0,0 +1,115 @@ +/* + * Filename: worker.c + * Author: fangdingjun@gmail.com + * License: GPLv2 + * Date: 2011-7-10 + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define fill_mem(p,d,s) memset(p,d,s) +#include "qpi.h" + +int read_mem(char *p, int data, size_t s); + +int worker(int num) +{ + + char *local, *remote; + //int id; + int k; + + fprintf(stderr, "\tbind process %d to cpu %ld\n", num, + cpu_num - num - 1); + if (bind_to_cpu(cpu_num - num - 1) < 0) { + //(*total_e)++; + kill(0, SIGUSR1); + exit(-1); + } + local = (char *)create_shm(key[num], shm_size); + if (local == NULL) { + kill(0, SIGUSR1); + exit(-1); + } + k = (num % 2 == 0 ? num + 1 : num - 1); + + remote = (char *)wait_remote_shm(key[k], shm_size); + if (remote == NULL) { + kill(0, SIGUSR1); + exit(-1); + } + while (1) { + sem_wait(sem[num]); + fill_mem(local, data, shm_size); + sem_wait(write_lock); + (*total_w)++; + sem_post(write_lock); + sem_post(sem[num]); + sem_wait(sem[k]); + if (read_mem(remote, data, shm_size) < 0) { + sem_post(sem[k]); + (*total_e)++; + kill(0, SIGUSR1); + exit(-1); + } + sem_wait(read_lock); + (*total_r)++; + sem_post(read_lock); + sem_post(sem[k]); + } +} + +int read_mem(char *p, int data, size_t s) +{ + int buf_size = 4 << 20; // 4M + char buf[buf_size]; + int i; + int loop; + int remain = 0; + char *p1; + memset(buf, data, buf_size); + loop = s / buf_size; + if (s % buf_size != 0) { + remain = s % buf_size; + } + + p1 = p; + for (i = 0; i < loop; i++) { + if (memcmp(p1, buf, buf_size)) { // data not equal + return -1; + } + p1 += buf_size; + } + if (remain) { + if (memcmp(p1, buf, remain)) { + return -1; + } + } + return 0; +} + +int do_fork(int m) +{ + pid_t pid; + pid = fork(); + if (pid < 0) { + perror("fork"); + //kill(0,SIGUSR1); + return -1; + //sleep(1); + } else if (pid == 0) { // child + worker(m); + exit(0); + } + //parent + return 0; +} +