Centos 6.5 MD1 full disk space

Question

I have almost full disk space on md1

[root@server ~]# df -h
Filesystem      Size  Used Avail Use% Mounted on
/dev/md1         20G   18G  645M  97% /
tmpfs           7.8G     0  7.8G   0% /dev/shm
/dev/md0        283M   33M  236M  12% /boot
/dev/md2        897G  795M  851G   1% /data

In md2 /data directory I have a lot of free space, how do I switch free space from md2 to md1 without going into rescue mode?

[root@server /]# cd /
[root@server /]# du -sch * | grep M
7.6M    bin
31M     boot
723M    data
29M     etc
468M    home
157M    lib
23M     lib64
du: cannot access `proc/22430/task/22430/fd/4': No such file or directory
du: cannot access `proc/22430/task/22430/fdinfo/4': No such file or directory
du: cannot access `proc/22430/fd/4': No such file or directory
du: cannot access `proc/22430/fdinfo/4': No such file or directory
8.0M    root
14M     sbin
5.3M    tmp
[root@server /]# cd /var/log
[root@server log]# du -sch * | grep M
26M     audit
1.4M    btmp-20150301
11M     dovecot-info.log
476M    exim
3.7M    lfd.log
4.9M    maillog
4.6M    messages
1.1M    messages-20150208
7.0M    messages-20150215
5.6M    messages-20150222
5.7M    messages-20150301
4.4M    pureftpd.log
21M     sa
1.1M    secure-20150215
13M     secure-20150222
588M    total
[root@server log]# cd /usr
[root@server usr]# du -sch * | grep M
35M     bin
17M     include
110M    lib
91M     lib64
24M     libexec
37M     sbin
302M    share

Edited 2

[root@server ~]# lsof +L1
COMMAND    PID      USER   FD   TYPE DEVICE SIZE/OFF NLINK   NODE NAME
mdadm     1411      root  txt    REG    9,1   467440     0    161 /sbin/mdadm (deleted)
hald      1568 haldaemon  txt    REG    9,1   351168     0 266999 /usr/sbin/hald (deleted)
hald-runn 1569      root  txt    REG    9,1    16928     0 266960 /usr/libexec/hald-runner (deleted)
hald-addo 1601      root  txt    REG    9,1    24744     0 266941 /usr/libexec/hald-addon-input.#prelink#.nkT9K9 (deleted)
hald-addo 1617 haldaemon  txt    REG    9,1    15296     0 266937 /usr/libexec/hald-addon-acpi.#prelink#.4NsGVw (deleted)
mysqld_sa 1680      root  txt    REG    9,1   904872     0 392386 /bin/bash (deleted)
mysqld    1934     mysql    4u   REG    9,1        0     0 917704 /tmp/ibrkEfYt (deleted)
mysqld    1934     mysql    5u   REG    9,1        0     0 917705 /tmp/ibG7f1i1 (deleted)
mysqld    1934     mysql    6u   REG    9,1        0     0 917706 /tmp/ibG12MDy (deleted)
mysqld    1934     mysql    7u   REG    9,1        0     0 917707 /tmp/ibCjMu25 (deleted)
mysqld    1934     mysql   11u   REG    9,1        0     0 917708 /tmp/ib90nO6D (deleted)
dovecot   1964      root  txt    REG    9,1    66088     0 280950 /usr/sbin/dovecot (deleted)
dovecot   1964      root   96u   REG    9,1        0     0 653985 /var/run/dovecot/login-master-notify398b00503e1dc567 (deleted)
dovecot   1964      root  111u   REG    9,1        0     0 660088 /var/run/dovecot/login-master-notify1ede7cc8b66b8ff2 (deleted)
anvil     1985   dovecot  txt    REG    9,1    19816     0 280921 /usr/libexec/dovecot/anvil (deleted)
log       1986      root  txt    REG    9,1    17288     0 280939 /usr/libexec/dovecot/log (deleted)
php-fpm   2112      root    3u   REG    9,1        0     0 920999 /tmp/.ZendSem.zjwiD4 (deleted)
php-fpm   2113  tajneala    3u   REG    9,1        0     0 920999 /tmp/.ZendSem.zjwiD4 (deleted)
php-fpm   2114  tajneala    3u   REG    9,1        0     0 920999 /tmp/.ZendSem.zjwiD4 (deleted)
php-fpm   2115  tajneala    3u   REG    9,1        0     0 920999 /tmp/.ZendSem.zjwiD4 (deleted)
php-fpm   2116  tajneala    3u   REG    9,1        0     0 920999 /tmp/.ZendSem.zjwiD4 (deleted)
php-fpm   2117  tajneala    3u   REG    9,1        0     0 920999 /tmp/.ZendSem.zjwiD4 (deleted)
php-fpm   2118  tajneala    3u   REG    9,1        0     0 920999 /tmp/.ZendSem.zjwiD4 (deleted)
mingetty  2186      root  txt    REG    9,1    15256     0     82 /sbin/mingetty (deleted)
mingetty  2188      root  txt    REG    9,1    15256     0     82 /sbin/mingetty (deleted)
mingetty  2190      root  txt    REG    9,1    15256     0     82 /sbin/mingetty (deleted)
mingetty  2192      root  txt    REG    9,1    15256     0     82 /sbin/mingetty (deleted)
mingetty  2194      root  txt    REG    9,1    15256     0     82 /sbin/mingetty (deleted)
mingetty  2196      root  txt    REG    9,1    15256     0     82 /sbin/mingetty (deleted)
nrsysmond 5789  newrelic  txt    REG    9,1  4654616     0 281503 /usr/sbin/nrsysmond (deleted)
nrsysmond 5790  newrelic  txt    REG    9,1  4654616     0 281503 /usr/sbin/nrsysmond (deleted)

Edit 3

[root@server ~]# cd /
[root@server /]# du -sch * | grep G
du: cannot access `proc/29259/task/29259/fd/4': No such file or directory
du: cannot access `proc/29259/task/29259/fdinfo/4': No such file or directory
du: cannot access `proc/29259/fd/4': No such file or directory
du: cannot access `proc/29259/fdinfo/4': No such file or directory
16G     usr
1.2G    var
19G     total
[root@server /]# cd /var/log
[root@server log]# du -sch * | grep G
[root@server log]# cd /usr
[root@server usr]# du -sch * | grep G
16G     local
16G     total
[root@server usr]#

Edit 4

[root@server ~]# du -a /usr | sort -n -r | head -n 20
16370388        /usr
15742324        /usr/local
15491028        /usr/local/apps
14599360        /usr/local/apps/apache
14569328        /usr/local/apps/apache/logs
14184552        /usr/local/apps/apache/logs/myDOMAIN.com.log
380100  /usr/local/apps/apache/logs/suphp55_log
309236  /usr/share
290736  /usr/local/apps/mysql
201760  /usr/local/apps/php55
179020  /usr/local/emps
155700  /usr/local/apps/lib
145100  /usr/local/apps/mysql/bin
129960  /usr/local/apps/mysql/share
124440  /usr/local/apps/mysql/share/mysql-test
112252  /usr/lib
96800   /usr/lib/locale
96796   /usr/lib/locale/locale-archive
95936   /usr/local/apps/mysql/share/mysql-test/suite
95416   /usr/local/apps/python2

Actually I just realised you have a grep M in there, you're probably missing stuff that's G sized. — user9517, Mar 06 '15 at 19:24

score 1 · Answer 1 · edited Apr 13 '17 at 12:14

1

First you need to determine what is filling the / filesystem up, search SF for linux full disk. Once you know what is causing the problem, then you will be able to focus on a solution. The chances are high though that it is log files in /var/log that are getting too big and need pruning, you may get away with configuring logrotate appropriately, you may have to move /var which is non trivial.

Ultimately what you find when you look for what is filling the disk will determine whether you have to use rescue mode or not but it seems unlikely you'll be lucky.

Looking at your edit it seems likely that you have an unlinked open file take a look at the output from lsof +L1.

And based upon the latest edit I would say that I was correct about you needing logrotate the /usr/local/apps/apache/logs/myDOMAIN.com.log file needs managing.

edited Apr 13 '17 at 12:14

Community

1

answered Mar 06 '15 at 18:47

user9517

114,104
20
206
289

Thank you,what you suggest to do? – 1809 Mar 06 '15 at 19:52
I suggest you read up on logrotate. – user9517 Mar 06 '15 at 21:19
I`m opened logrotate conf file ,how there to resolve problem?As i see there i can setup log files,logs backups,logs compress...How to get more free space?My disk is 98% full – 1809 Mar 06 '15 at 23:34

Centos 6.5 MD1 full disk space

1 Answers1