From daac7fda6555f7fa283df3fab52cbb6b406e5a56 Mon Sep 17 00:00:00 2001 From: fangdingjun Date: Sat, 16 Jul 2011 12:49:08 +0800 Subject: [PATCH] initial project --- .gitignore | 20 ++++ AUTHORS | 1 + ChangeLog | 3 + Makefile.am | 1 + NEWS | 3 + README | 30 +++++ autogen.sh | 2 + configure.ac | 28 +++++ src/Makefile.am | 4 + src/cpu.c | 47 ++++++++ src/main.c | 302 ++++++++++++++++++++++++++++++++++++++++++++++++ src/qpi.h | 33 ++++++ src/report.c | 114 ++++++++++++++++++ src/shm.c | 131 +++++++++++++++++++++ src/worker.c | 115 ++++++++++++++++++ 15 files changed, 834 insertions(+) create mode 100644 .gitignore create mode 100644 AUTHORS create mode 100644 ChangeLog create mode 100644 Makefile.am create mode 100644 NEWS create mode 100644 README create mode 100755 autogen.sh create mode 100644 configure.ac create mode 100644 src/Makefile.am create mode 100644 src/cpu.c create mode 100644 src/main.c create mode 100644 src/qpi.h create mode 100644 src/report.c create mode 100644 src/shm.c create mode 100644 src/worker.c diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c6e1c91 --- /dev/null +++ b/.gitignore @@ -0,0 +1,20 @@ +*~ +*.o +COPYING +INSTALL +Makefile.in +aclocal.m4 +autom4te.cache +autoscan-2.68.log +compile +config.* +configure +depcomp +install-sh +missing +Makefile +src/.deps +src/qpi_stress +stamp-h1 +cscope* + diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..7810370 --- /dev/null +++ b/AUTHORS @@ -0,0 +1 @@ +fangdingjun diff --git a/ChangeLog b/ChangeLog new file mode 100644 index 0000000..2a7c725 --- /dev/null +++ b/ChangeLog @@ -0,0 +1,3 @@ + +2011-7-16: +initial project diff --git a/Makefile.am b/Makefile.am new file mode 100644 index 0000000..af437a6 --- /dev/null +++ b/Makefile.am @@ -0,0 +1 @@ +SUBDIRS = src diff --git a/NEWS b/NEWS new file mode 100644 index 0000000..5a1387d --- /dev/null +++ b/NEWS @@ -0,0 +1,3 @@ + +2011-7-16 +initial projects diff --git a/README b/README new file mode 100644 index 0000000..4fcb45c --- /dev/null +++ b/README @@ -0,0 +1,30 @@ + +1.overview + + this is a tool to test CPU's QPI channel: + + start a process A and bind to cpu0, then request a share memory + start a process B and bind to cpu1, then attach to process A's share memory + process A write to the share memory, process B read from then share memory via QPI channel + process A read the process B's share memory via QPI channel + + procA procB + || || + +------+ +--------+ + |cpu0 | QPI channel | cpu1 | + +------+ +--------+ + || || + +-----+ +-----+ + |memA | |memB | + +-----+ +-----+ + +2. install + unpack the source and enter the source directory, type + ./configure + make + make install + +3.usage + type + qpi_stress -h + for help diff --git a/autogen.sh b/autogen.sh new file mode 100755 index 0000000..7db7e5c --- /dev/null +++ b/autogen.sh @@ -0,0 +1,2 @@ +#!/bin/sh +aclocal && autoheader && autoconf && automake -a && ./configure diff --git a/configure.ac b/configure.ac new file mode 100644 index 0000000..3f9743c --- /dev/null +++ b/configure.ac @@ -0,0 +1,28 @@ +# -*- Autoconf -*- +# Process this file with autoconf to produce a configure script. + +AC_PREREQ([2.68]) +AC_INIT([qpi-stress], [0.1], [fangdingjun@gmail.com]) +AC_CONFIG_SRCDIR([src/worker.c]) +AC_CONFIG_HEADERS([config.h]) +AM_INIT_AUTOMAKE(1.0) + +# Checks for programs. +AC_PROG_CC +AM_PROG_CC_C_O + +# Checks for libraries. + +# Checks for header files. +AC_CHECK_HEADERS([fcntl.h stdlib.h string.h sys/time.h unistd.h]) + +# Checks for typedefs, structures, and compiler characteristics. +AC_TYPE_PID_T +AC_TYPE_SIZE_T + +# Checks for library functions. +AC_FUNC_FORK +AC_CHECK_FUNCS([alarm gettimeofday memset]) +AC_CONFIG_FILES([Makefile + src/Makefile]) +AC_OUTPUT diff --git a/src/Makefile.am b/src/Makefile.am new file mode 100644 index 0000000..6647d72 --- /dev/null +++ b/src/Makefile.am @@ -0,0 +1,4 @@ +bin_PROGRAMS = qpi_stress +qpi_stress_SOURCES = main.c cpu.c shm.c report.c worker.c +qpi_stress_CFLAGS = -Wall -Werror -D_GNU_SOURCE +qpi_stress_LDFLAGS = -lpthread diff --git a/src/cpu.c b/src/cpu.c new file mode 100644 index 0000000..bb4f738 --- /dev/null +++ b/src/cpu.c @@ -0,0 +1,47 @@ +/* + * Filename: cpu.c + * Author: fangdingjun@gmail.com + * License: GPLv2 + * Date: 2011-7-10 + * + */ + +#include +#include +#include +//#include "qpi.h" + +int bind_to_cpu(int cpunum) +{ + int num = sysconf(_SC_NPROCESSORS_CONF); + int i; + + cpu_set_t mask; + + CPU_ZERO(&mask); + CPU_SET(cpunum, &mask); + + if (sched_setaffinity(0, sizeof(mask), &mask) == -1) { + perror("set cpu affinity failed"); + return -1; + } + + CPU_ZERO(&mask); + if (sched_getaffinity(0, sizeof(mask), &mask) == -1) { + perror("get cpu affinity failed"); + return -1; + } + for (i = 0; i < num; i++) { + if (CPU_ISSET(i, &mask)) { + break; + } + } + if (i != cpunum) { + fprintf(stderr, + "set cpu affinity failed: process not running on cpu %d\n", + cpunum); + return -1; + } + return 0; +} + diff --git a/src/main.c b/src/main.c new file mode 100644 index 0000000..5659c4b --- /dev/null +++ b/src/main.c @@ -0,0 +1,302 @@ +/* + * this is tool to stress cpu's qpi channel on romely platform + * + * Filename: main.c + * Author: fangdingjun@gmail.com + * License: GPLv2 + * Date: 2011-7-10 + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "qpi.h" + + +int main(int argc, char **argv) +{ + /* initial global value */ + proc_num = 2; + data = 0xaa; + shm_size = 4 << 20; // 4 MB + running = 1; + output = NULL; + fp = NULL; + ret = 0; + + int opt_idx; + unsigned int timeout = 0; + int delay = 10; + long page_size; + long total_page; + unsigned long req_mem; + unsigned long avb_mem; + int c; + int i; + key_t k_read; + key_t k_write; + key_t k_error; + //int id_r,id_w; + + struct option opt[] = { + {"timeout", 1, 0, 't'}, + {"number", 1, 0, 'n'}, + {"data", 1, 0, 'd'}, + {"size", 1, 0, 's'}, + {"help", 0, 0, 'h'}, + {"delay", 1, 0, 'p'}, + {"output", 1, 0, 'o'}, + {0, 0, 0, 0} + }; + proc_num = cpu_num = sysconf(_SC_NPROCESSORS_CONF); + // process command line options + while (1) { + c = getopt_long(argc, argv, "hn:t:d:s:p:o:", opt, &opt_idx); + if (c == -1) { + break; + } + switch (c) { + case 'n': + proc_num = atoi(optarg); + break; + case 't': + timeout = atoi(optarg) * 60; + break; + case 'd': + //data=atoi(optarg); + if (!memcmp("0x", optarg, 2)) { + sscanf(optarg, "%x", &data); + } else { + data = atoi(optarg); + } + break; + case 's': + shm_size = atoi(optarg) << 20; + break; + case 'p': + delay = atoi(optarg); + break; + case 'o': + output = optarg; + break; + case 'h': + usage(); + break; + default: + usage(); + } + } + if (argc > optind) { + usage(); + } + // process number error + page_size = sysconf(_SC_PAGESIZE); + total_page = sysconf(_SC_PHYS_PAGES); + req_mem = proc_num * shm_size; + avb_mem = page_size * total_page; + //fp=stderr; + if (output != NULL) { + fp = fopen(output, "w"); + if (fp == NULL) { + fprintf(stderr, + "WARNING: open %s failed, log to screen\n", + output); + //fp=stderr; + } + } + // 0 for random data + if (data == 0) { + srand(time(NULL)); + data = rand() % 0xff; + } + //printf("argc %d optind %d\n",argc,optind); + //return 0; + fprintf(stdout, "summay:\n" + "\ttotal processors: %ld\n" + "\ttotal memory: %ld MB\n" + "\tstart processes: %d\n" + "\tdata partern: 0x%02x\n" + "\truntime: %u seconds\n" + "\tresult display delay: %d seconds\n" + "\tper process share memory size: %lu MB\n\n", + cpu_num, + avb_mem >> 20, + proc_num, data, + timeout == + 0 ? (unsigned int)-1 : (unsigned int)timeout, delay, + shm_size >> 20); + + if (proc_num % 2) { + fprintf(stderr, + "ERROR: process number must be 2, 4, 6, 8, ...\n"); + exit(-1); + } + + if (proc_num > cpu_num) { + fprintf(stderr, + "ERROR: only %ld processors core," + " but you want to run %d processes\n", + cpu_num, proc_num); + exit(-1); + } + + if (req_mem > avb_mem) { + fprintf(stderr, + "ERROR: total available memory is %ld bytes," + " but you request %d * %lu = %lu bytes\n", + avb_mem, proc_num, shm_size, req_mem); + exit(-1); + } + + sem_t *l_sem[proc_num]; + sem = l_sem; + key_t a_key[proc_num]; + key = a_key; + init_shm_key(key, proc_num); + init_sem(sem, proc_num); + init_one_key(&k_read, "/dev/shm/read_counter"); + init_one_key(&k_write, "/dev/shm/write_counter"); + init_one_key(&k_error, "/dev/shm/error_counter"); + total_r = (int *)create_shm(k_read, sizeof(int)); + total_w = (int *)create_shm(k_write, sizeof(int)); + total_e = (int *)create_shm(k_error, sizeof(int)); + if (!(total_r && total_w && total_e)) { + //exit(-1); + ret = -1; + goto out; + } + *total_e = 0; + *total_r = 0; + *total_w = 0; + + read_lock = sem_open("read", O_CREAT, 0644, 0); + if (read_lock == SEM_FAILED) { + perror("sem_open"); + //exit(-1); + ret = -1; + goto out; + } + sem_init(read_lock, 1, 0); + + write_lock = sem_open("write", O_CREAT, 0644, 0); + if (write_lock == SEM_FAILED) { + perror("sem_open"); + //exit(-1); + ret = -1; + goto out; + } + sem_init(write_lock, 1, 0); + + for (i = 0; i < proc_num; i++) { + if (do_fork(i) < 0) { // fork subprocess + ret = -1; + goto out1; + } + } + + usleep(100); + fprintf(stdout, "-------------------------------\n"); + fprintf(stdout, "\nQPI stress staring ...\n\n"); + //sleep(1); + + signal(SIGINT, sig); + signal(SIGTERM, sig); + signal(SIGHUP, sig); + signal(SIGALRM, sig); + signal(SIGUSR1, sig); + signal(SIGUSR2, sig); + + // register timer + if (timeout) { + alarm(timeout); + } + //report status + report(delay); + +out1: + kill(0, SIGUSR2); // terminate children process + sleep(2); + + // check if data check error + if (*total_e) { + fprintf(stderr, "data check error: %d\n", *total_e); + ret = -1; + } +out: + // destroy share memory and semaphore + destroy_shm(key, proc_num); + destroy_sem(sem, proc_num); + + /* cleaning semaphore */ + sem_close(write_lock); + sem_close(read_lock); + sem_unlink("write"); + sem_unlink("read"); + + delete_shm_by_name("/dev/shm/read_counter", sizeof(int)); + delete_shm_by_name("/dev/shm/write_counter", sizeof(int)); + delete_shm_by_name("/dev/shm/error_counter", sizeof(int)); + + if(fp){ + fclose(fp); + } + return ret; +} + +int delete_shm_by_name(char *n, size_t s) +{ + key_t k; + int id; + k = ftok(n, 0); + id = shmget(k, s, 0644); + if (id != -1) { + shmctl(id, IPC_RMID, NULL); + } + unlink(n); + return 0; +} + +void usage() +{ + fprintf(stderr, "Usage: " + "qpi_stress [options]" + "\n\t-h, --help\n\t\tprint this help" + "\n\t-n, --number=n\n\t\t start n processes, default number of cpu cores\n" + "\t-t, --timeout=n\n\t\t time to run(minute), default 0-run util CTRL+C pressed\n" + "\t-d, --data=n\n\t\t data to fill memory, 0 for random, default 0xaa\n" + "\t-s, --size=n\n\t\t size of share memory(MB), default 4\n" + "\t-p, --delay=n\n\t\t result display delay(seconds), default 10\n" + "\t-o, --output=file\n\t\t write result to file, default screen\n" + "Example:\n\t" + "qpi_stress -n4 -s10 -t10 -d 0x55" + "\n start 4 processes, run 10 minutes, per process requst 10 MB memory, use 0x55 to fill\n"); + exit(-1); +} + +void sig(int num) +{ + //printf("get signal %d, cleaning and exit ...\n",num); + if (num == SIGALRM) { + fprintf(stderr, "timeout, cleaning and exiting...\n"); + } else if (num == SIGINT) { + fprintf(stderr, "CTRL+C pressed, aborting...\n"); + } else if (num == SIGUSR1) { + fprintf(stderr, "error occured, aborting...\n"); + ret = 1; + } else if (num == SIGUSR2) { + ; + } else { + fprintf(stderr, "got signal %d, exiting...\n", num); + } + running = 0; + //return; +} diff --git a/src/qpi.h b/src/qpi.h new file mode 100644 index 0000000..66adc8b --- /dev/null +++ b/src/qpi.h @@ -0,0 +1,33 @@ +#ifndef QPI_H +#define QPI_H + sem_t *read_lock; + sem_t *write_lock; + int proc_num; + unsigned int data; + long cpu_num; + int *total_r; + int *total_w; + int *total_e; + key_t *key; + sem_t **sem; + unsigned long shm_size; + volatile int running; + char *output; + FILE *fp; + void usage(); + int init_shm_key(key_t * k, int num); + int init_sem(sem_t ** s, int num); + int init_one_key(key_t * k, char *name); + int do_fork(int m); + int worker(int num); + int destroy_shm(key_t * k, int num); + int destroy_sem(sem_t ** s, int num); + int report(int delay); + int delete_shm_by_name(char *n, size_t s); + int ret; + void sig(int num); + void *create_shm(key_t k, size_t s); + int bind_to_cpu(int cpunum); + void *create_shm(key_t k, size_t s); + void *wait_remote_shm(key_t k, size_t s); +#endif diff --git a/src/report.c b/src/report.c new file mode 100644 index 0000000..e396ce9 --- /dev/null +++ b/src/report.c @@ -0,0 +1,114 @@ +/* + * Filename: report.c + * Author: fangdingjun@gmail.com + * License: GPLv2 + * Date: 2011-7-10 + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include "qpi.h" + +#define DIFF(start,end) ((end.tv_sec-start.tv_sec) * 1000 + (end.tv_usec-start.tv_usec)) + +int report(int delay) +{ + struct timeval tv, tv1; + unsigned long tmp_read = 0, tmp_write = 0; + unsigned long timedelay = 0; + unsigned long mem_total_r = 0, mem_total_w = 0; + time_t now; + struct tm *tt; + char t_out[100]; + int i; + + + if (fp){ + fprintf(fp, "0 Read_perf_MB/s Read_delay_ms " + "Write_perf_MB/s Write_delay_ms\n"); + } + + // wake up thread + for (i = 0; i < proc_num; i++) { + //printf("sem_post(&shm_sem[%d])\n",i); + sem_post(sem[i]); + } + + sem_post(read_lock); + sem_post(write_lock); + gettimeofday(&tv, NULL); + while (running) { + memcpy(&tv1, &tv, sizeof(struct timeval)); + gettimeofday(&tv, NULL); + + timedelay = DIFF(tv1, tv); + sem_wait(read_lock); + tmp_read = *total_r; + *total_r = 0; + sem_post(read_lock); + + sem_wait(write_lock); + tmp_write = *total_w; + *total_w = 0; + sem_post(write_lock); + + //printf("timedelay %lu, read = %lu,write=%lu\n",timedelay,tmp_read,tmp_write); + + now = time(NULL); + tt = localtime(&now); + if (tmp_read && tmp_write && timedelay) { + mem_total_r = (tmp_read * shm_size) >> 20; + mem_total_w = (tmp_write * shm_size) >> 20; + //printf("%s, read %lu MB," " write %lu MB," + // " per %d seconds\n", + //ctime(&now), + // t_out, mem_total_r, mem_total_w, delay); + strftime(t_out, sizeof(t_out), "%Y-%m-%d %H:%M:%S", tt); + if (!fp) { + fprintf(stdout, "in recent %lu r/w(s):\n" + "%s\n" + "read: %4.2f MB/s, delay: %4.2f ms, write: %4.2f MB/s, delay: %4.2f ms\n\n", + tmp_read + tmp_write, + t_out, + (float)mem_total_r * 1000 / + (float)timedelay, + (float)timedelay / (float)tmp_read, + (float)mem_total_w * 1000 / + (float)timedelay, + (float)timedelay / (float)tmp_write); + + } else { + strftime(t_out, sizeof(t_out), "%m_%d_%H:%M:%S", + tt); + fprintf(fp, "%s %4.2f %4.2f %4.2f %4.2f\n", + t_out, + (float)mem_total_r * 1000 / + (float)timedelay, + (float)timedelay / (float)tmp_read, + (float)mem_total_w * 1000 / + (float)timedelay, + (float)timedelay / (float)tmp_write); + } + } + //total_old_r = total_r; + //total_old_w = total_w; + // if error, stop it + if (*total_e) { + //printf("data check error: %d\n",error); + kill(0, SIGUSR1); + break; + } + //printf("total=%d\n",total); + //gettimeofday(&tv1,NULL); + sleep(delay); + } + //fclose(fp); + return 0; +} + diff --git a/src/shm.c b/src/shm.c new file mode 100644 index 0000000..2f3c1c3 --- /dev/null +++ b/src/shm.c @@ -0,0 +1,131 @@ +/* + * Filename: report.c + * Author: fangdingjun@gmail.com + * License: GPLv2 + * Date: 2011-7-10 + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include "qpi.h" + +void *create_shm(key_t k, size_t s) +{ + int id; + void *p; + id = shmget(k, s, IPC_CREAT | 0644); + if (id == -1) { + perror("shmget"); + return NULL; + } + p = shmat(id, NULL, 0); + if (p == (void *)-1) { + perror("shmat"); + return NULL; + } + return p; +} + +int init_one_key(key_t * k, char *name) +{ + FILE *fp = fopen(name, "w"); + fclose(fp); + *k = ftok(name, 0); + return 0; +} + +int init_shm_key(key_t * k, int num) +{ + int i; + char fn_r[] = "/dev/shm/qpitestxxxxx"; + char fname[256]; + for (i = 0; i < num; i++) { + sprintf(fname, "%s%d", fn_r, i); + FILE *fp = fopen(fname, "w"); + fclose(fp); + k[i] = ftok(fname, 0); + } + return 0; +} + +int destroy_shm(key_t * k, int num) +{ + int i; + int id; + char fn_r[] = "/dev/shm/qpitestxxxxx"; + char fname[256]; + for (i = 0; i < num; i++) { + id = shmget(k[i], shm_size, 0644); + if (id != -1) { + shmctl(id, IPC_RMID, NULL); + } + sprintf(fname, "%s%d", fn_r, i); + unlink(fname); + } + return 0; +} + +int destroy_sem(sem_t ** s, int num) +{ + int i; + char sem_r[] = "semxxxx"; + char sem_n[256]; + for (i = 0; i < num; i++) { + sem_close(s[i]); + sprintf(sem_n, "%s%d", sem_r, i); + sem_unlink(sem_n); + } + return 0; +} + +int init_sem(sem_t ** s, int num) +{ + int i; + char sem_r[] = "semxxxx"; + char sem_n[256]; + for (i = 0; i < num; i++) { + sprintf(sem_n, "%s%d", sem_r, i); + s[i] = sem_open(sem_n, O_CREAT, 0644, 0); + if (s[i] == SEM_FAILED) { + perror("sem_open"); + exit(-1); + } + sem_init(s[i], 1, 0); + } + return 0; +} + +void *wait_remote_shm(key_t k, size_t s) +{ + void *p; + int id; + while (1) { // wait remote to initial share memory + id = shmget(k, s, 0644); + if (id == -1) { + if (errno == ENOENT) { + usleep(10); + } else { + perror("shmget"); + //exit(-1); + return NULL; + } + } else { + break; + } + } + + p = shmat(id, NULL, 0); + if (p == (void *)-1) { + perror("shmat"); + //exit(-1); + return NULL; + } + return p; +} + diff --git a/src/worker.c b/src/worker.c new file mode 100644 index 0000000..355813b --- /dev/null +++ b/src/worker.c @@ -0,0 +1,115 @@ +/* + * Filename: worker.c + * Author: fangdingjun@gmail.com + * License: GPLv2 + * Date: 2011-7-10 + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define fill_mem(p,d,s) memset(p,d,s) +#include "qpi.h" + +int read_mem(char *p, int data, size_t s); + +int worker(int num) +{ + + char *local, *remote; + //int id; + int k; + + fprintf(stderr, "\tbind process %d to cpu %ld\n", num, + cpu_num - num - 1); + if (bind_to_cpu(cpu_num - num - 1) < 0) { + //(*total_e)++; + kill(0, SIGUSR1); + exit(-1); + } + local = (char *)create_shm(key[num], shm_size); + if (local == NULL) { + kill(0, SIGUSR1); + exit(-1); + } + k = (num % 2 == 0 ? num + 1 : num - 1); + + remote = (char *)wait_remote_shm(key[k], shm_size); + if (remote == NULL) { + kill(0, SIGUSR1); + exit(-1); + } + while (1) { + sem_wait(sem[num]); + fill_mem(local, data, shm_size); + sem_wait(write_lock); + (*total_w)++; + sem_post(write_lock); + sem_post(sem[num]); + sem_wait(sem[k]); + if (read_mem(remote, data, shm_size) < 0) { + sem_post(sem[k]); + (*total_e)++; + kill(0, SIGUSR1); + exit(-1); + } + sem_wait(read_lock); + (*total_r)++; + sem_post(read_lock); + sem_post(sem[k]); + } +} + +int read_mem(char *p, int data, size_t s) +{ + int buf_size = 4 << 20; // 4M + char buf[buf_size]; + int i; + int loop; + int remain = 0; + char *p1; + memset(buf, data, buf_size); + loop = s / buf_size; + if (s % buf_size != 0) { + remain = s % buf_size; + } + + p1 = p; + for (i = 0; i < loop; i++) { + if (memcmp(p1, buf, buf_size)) { // data not equal + return -1; + } + p1 += buf_size; + } + if (remain) { + if (memcmp(p1, buf, remain)) { + return -1; + } + } + return 0; +} + +int do_fork(int m) +{ + pid_t pid; + pid = fork(); + if (pid < 0) { + perror("fork"); + //kill(0,SIGUSR1); + return -1; + //sleep(1); + } else if (pid == 0) { // child + worker(m); + exit(0); + } + //parent + return 0; +} +