This is a multi-part message in MIME format.
--------------319B4EC131B0C6B82BB716D7
Content-Type: text/plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Here are some hard numbers on mmap vs read/write on Linux.
All tests are using blocks of 8KB and datafile of 1MB. Test programs
attached.
--- file creation ---
mmap-file: mmap() of new file
18.38user 11.52system 0:39.26elapsed (51261major+8minor)pagefaults
write-file: write() to new file
15.95user 5.80system 0:32.84elapsed (79major+10minor)pagefaults
--- file -> socket ---
mmap-socket: mmap() file, write() to socket
0.07user 7.88system 0:34.92elapsed (51264major+8minor)pagefaults
read-socket: read() from file, write() to socket
0.13user 9.02system 0:22.17elapsed (25663major+10minor)pagefaults
--- socket -> file ---
socket-mmap: read() from socket to mmap()ed file
0.30user 13.39system 0:57.83elapsed (51266major+8minor)pagefaults
socket-write: read() from a socket, write() to file
0.12user 9.21system 0:45.66elapsed (77major+10minor)pagefaults
----------------------
All file read (both read() and mmap) tests showed identical memory
behaviour: freepages pushed down to min_free, increased VM cache, no
paging.
mmap()ed writes apparently uses both VM cache and the FS buffer cache.
freepages pushed down to min_free. no paging.
File write() showed a different memory usage. Only the FS buffer cache
used, freepages stayed at min_free*2, and there was some paging
(probably from VM system trying to get freepages abovefree_pages_high).
This is probably a sign that the buffer cache memory management in Linux
2.0 could do better. This paging is probably only a problem when there
is a high amount of write, and very few reads (the VM cache needs to be
drained before paging occurs).
All measurements was taken from the second run of each program. On the
file creation tests (all except *-socket) the test file was removed
before each run to avoid measuring the deallocation of file blocks.
It looks like I have to revise my thoughts on mmap vs read/write a
little bit, but only a little. Using mmap() in a VM friendly way removes
most of the benefits from using mmap(). The benefits left is only
somewhat lower CPU usage when writing directly to a socket from a file,
and the possibility to have multiple outstanding I/O operations on a
single filedescriptor when using threads.
I do agree with Alex that we would probably gain much by analysing how
Squid transfers data back and forth if CPU usage is a problem (which I
don't think it is yet).
/Henrik
--------------319B4EC131B0C6B82BB716D7
Content-Type: text/plain; charset=us-ascii; name="mmap-file.c"
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="mmap-file.c"
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <unistd.h>
#include <fcntl.h>
#include <assert.h>
void main(int argc, char **argv)
{
int fd;
int file_size=100*1024*1024;
int buf_size=8192;
char *buf;
int i,j;
assert(argc>1);
fd=open(argv[1],O_RDWR|O_CREAT|O_TRUNC,0666);
assert(fd>=0);
ftruncate(fd,file_size);
for(i=0;i<file_size;) {
buf=mmap(NULL,buf_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, i);
assert(buf!=MAP_FAILED);
for(j=0;i<file_size && j<buf_size;i++,j++)
buf[j]=(char)i;
munmap(buf, buf_size);
}
close(fd);
}
--------------319B4EC131B0C6B82BB716D7
Content-Type: text/plain; charset=us-ascii; name="mmap-socket.c"
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="mmap-socket.c"
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <fcntl.h>
#include <assert.h>
void main(int argc, char **argv)
{
int fd,s;
int file_size=100*1024*1024;
int buf_size=8192;
char *buf;
int i,j;
int ret;
struct sockaddr_in so;
so.sin_family=AF_INET;
inet_aton("127.0.0.1",&so.sin_addr);
so.sin_port=htons(9);
assert(argc>1);
s=socket(AF_INET, SOCK_STREAM, 0);
assert(s>=0);
if(connect(s,&so,sizeof(so)) != 0) {
perror("connect");
exit(1);
}
fd=open(argv[1],O_RDONLY,0666);
assert(fd>=0);
ftruncate(fd,file_size);
for(i=0;i<file_size;i+=buf_size) {
buf=mmap(NULL,buf_size, PROT_READ, MAP_SHARED, fd, i);
if (buf==MAP_FAILED) {
perror("mmap");
exit(2);
}
write(s,buf,buf_size);
munmap(buf, buf_size);
}
close(fd);
}
--------------319B4EC131B0C6B82BB716D7
Content-Type: text/plain; charset=us-ascii; name="read-socket.c"
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="read-socket.c"
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <fcntl.h>
#include <assert.h>
void main(int argc, char **argv)
{
int fd,s;
int file_size=100*1024*1024;
const int buf_size=8192;
char buf[buf_size];
int i,j;
int ret;
struct sockaddr_in so;
so.sin_family=AF_INET;
inet_aton("127.0.0.1",&so.sin_addr);
so.sin_port=htons(9);
assert(argc>1);
s=socket(AF_INET, SOCK_STREAM, 0);
assert(s>=0);
if(connect(s,&so,sizeof(so)) != 0) {
perror("connect");
exit(1);
}
fd=open(argv[1],O_RDONLY,0666);
assert(fd>=0);
ftruncate(fd,file_size);
for(i=0;i<file_size;i+=buf_size) {
j=read(fd,buf,buf_size);
assert(j==buf_size);
write(s,buf,buf_size);
}
close(s);
close(fd);
}
--------------319B4EC131B0C6B82BB716D7
Content-Type: text/plain; charset=us-ascii; name="socket-mmap.c"
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="socket-mmap.c"
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <fcntl.h>
#include <assert.h>
void main(int argc, char **argv)
{
int fd,s;
int file_size=100*1024*1024;
int buf_size=8192;
char *buf;
int i,j,len;
int ret;
struct sockaddr_in so;
so.sin_family=AF_INET;
inet_aton("127.0.0.1",&so.sin_addr);
so.sin_port=htons(19);
assert(argc>1);
s=socket(AF_INET, SOCK_STREAM, 0);
assert(s>=0);
if(connect(s,&so,sizeof(so)) != 0) {
perror("connect");
exit(1);
}
fd=open(argv[1],O_RDWR|O_TRUNC|O_CREAT,0666);
assert(fd>=0);
ftruncate(fd,file_size);
for(i=0;i<file_size;i+=j) {
buf=mmap(NULL,buf_size, PROT_WRITE, MAP_SHARED, fd, i);
if (buf==MAP_FAILED) {
perror("mmap");
exit(2);
}
for(j=0;j<buf_size;j+=len) {
len=read(s,buf+j,buf_size-j);
assert(len>0);
}
munmap(buf, buf_size);
}
close(fd);
}
--------------319B4EC131B0C6B82BB716D7
Content-Type: text/plain; charset=us-ascii; name="socket-write.c"
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="socket-write.c"
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <fcntl.h>
#include <assert.h>
void main(int argc, char **argv)
{
int fd,s;
int file_size=100*1024*1024;
const int buf_size=8192;
char buf[buf_size];
int i,j,len;
int ret;
struct sockaddr_in so;
so.sin_family=AF_INET;
inet_aton("127.0.0.1",&so.sin_addr);
so.sin_port=htons(19);
assert(argc>1);
s=socket(AF_INET, SOCK_STREAM, 0);
assert(s>=0);
if(connect(s,&so,sizeof(so)) != 0) {
perror("connect");
exit(1);
}
fd=open(argv[1],O_WRONLY|O_TRUNC|O_CREAT,0666);
assert(fd>=0);
ftruncate(fd,file_size);
for(i=0;i<file_size;i+=j) {
for(j=0;j<buf_size;j+=len) {
len=read(s,buf+j,buf_size-j);
assert(len>0);
}
write(fd, buf, j);
}
close(fd);
}
--------------319B4EC131B0C6B82BB716D7
Content-Type: text/plain; charset=us-ascii; name="write-file.c"
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="write-file.c"
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <unistd.h>
#include <fcntl.h>
#include <assert.h>
void main(int argc, char **argv)
{
int fd;
char buf[8192];
int file_size=100*1024*1024;
int i,j;
assert(argc>1);
fd=open(argv[1],O_RDWR|O_CREAT|O_TRUNC,0666);
assert(fd>=0);
ftruncate(fd,file_size);
for(i=0;i<file_size;) {
for(j=0;i<file_size && j<sizeof(buf);i++,j++)
buf[j]=(char)i;
write(fd,buf,j);
}
close(fd);
}
--------------319B4EC131B0C6B82BB716D7--
Received on Tue Jul 29 2003 - 13:15:50 MDT
This archive was generated by hypermail pre-2.1.9 : Tue Dec 09 2003 - 16:11:48 MST