1

Our developers wants bamboo because of integrations to other Atlassian software and we have regularly problems with this buildserver. Often the buildjobs fail with OutOfMemory exceptions like this:

 java.lang.OutOfMemoryError: unable to create new native thread
        at java.lang.Thread.start0(Native Method)
        at java.lang.Thread.start(Thread.java:717)
        at org.apache.tools.ant.taskdefs.ProcessDestroyer.removeShutdownHook(ProcessDestroyer.java:149)
        at org.apache.tools.ant.taskdefs.ProcessDestroyer.remove(ProcessDestroyer.java:202)
        at org.apache.tools.ant.taskdefs.Execute.execute(Execute.java:481)
        at org.apache.tools.ant.taskdefs.ExecTask.runExecute(ExecTask.java:629)
        at org.apache.tools.ant.taskdefs.ExecTask.runExec(ExecTask.java:670)
        at org.apache.tools.ant.taskdefs.ExecTask.execute(ExecTask.java:496)
        at org.apache.tools.ant.UnknownElement.execute(UnknownElement.java:293)
        at sun.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.tools.ant.dispatch.DispatchUtils.execute(DispatchUtils.java:106)
        at org.apache.tools.ant.Task.perform(Task.java:348)
        at org.apache.tools.ant.taskdefs.Sequential.execute(Sequential.java:68)
        at net.sf.antcontrib.logic.IfTask.execute(IfTask.java:197)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.tools.ant.dispatch.DispatchUtils.execute(DispatchUtils.java:106)
        at org.apache.tools.ant.TaskAdapter.execute(TaskAdapter.java:155)
        at org.apache.tools.ant.UnknownElement.execute(UnknownElement.java:293)
        at sun.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.tools.ant.dispatch.DispatchUtils.execute(DispatchUtils.java:106)
        at org.apache.tools.ant.Task.perform(Task.java:348)
        at org.apache.tools.ant.Target.execute(Target.java:435)
        at org.apache.tools.ant.Target.performTasks(Target.java:456)
        at org.apache.tools.ant.Project.executeSortedTargets(Project.java:1405)
        at org.apache.tools.ant.helper.SingleCheckExecutor.executeTargets(SingleCheckExecutor.java:38)
        at org.apache.tools.ant.Project.executeTargets(Project.java:1260)
        at org.apache.tools.ant.taskdefs.Ant.execute(Ant.java:441)
        at org.apache.tools.ant.taskdefs.CallTarget.execute(CallTarget.java:105)
        at org.apache.tools.ant.UnknownElement.execute(UnknownElement.java:293)
        at sun.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.tools.ant.dispatch.DispatchUtils.execute(DispatchUtils.java:106)
        at org.apache.tools.ant.Task.perform(Task.java:348)
        at org.apache.tools.ant.Target.execute(Target.java:435)
        at org.apache.tools.ant.Target.performTasks(Target.java:456)
        at org.apache.tools.ant.Project.executeSortedTargets(Project.java:1405)
        at org.apache.tools.ant.helper.SingleCheckExecutor.executeTargets(SingleCheckExecutor.java:38)
        at org.apache.tools.ant.Project.executeTargets(Project.java:1260)
        at org.apache.tools.ant.taskdefs.Ant.execute(Ant.java:441)
        at org.apache.tools.ant.taskdefs.CallTarget.execute(CallTarget.java:105)
        at org.apache.tools.ant.UnknownElement.execute(UnknownElement.java:293)
        at sun.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.tools.ant.dispatch.DispatchUtils.execute(DispatchUtils.java:106)
        at org.apache.tools.ant.Task.perform(Task.java:348)
        at org.apache.tools.ant.taskdefs.Sequential.execute(Sequential.java:68)
        at net.sf.antcontrib.logic.IfTask.execute(IfTask.java:197)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.tools.ant.dispatch.DispatchUtils.execute(DispatchUtils.java:106)
        at org.apache.tools.ant.TaskAdapter.execute(TaskAdapter.java:155)
        at org.apache.tools.ant.UnknownElement.execute(UnknownElement.java:293)
        at sun.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.tools.ant.dispatch.DispatchUtils.execute(DispatchUtils.java:106)
        at org.apache.tools.ant.Task.perform(Task.java:348)
        at org.apache.tools.ant.Target.execute(Target.java:435)
        at org.apache.tools.ant.Target.performTasks(Target.java:456)
        at org.apache.tools.ant.Project.executeSortedTargets(Project.java:1405)
        at org.apache.tools.ant.Project.executeTarget(Project.java:1376)
        at org.apache.tools.ant.helper.DefaultExecutor.executeTargets(DefaultExecutor.java:41)
        at org.apache.tools.ant.Project.executeTargets(Project.java:1260)
        at org.apache.tools.ant.Main.runBuild(Main.java:857)
        at org.apache.tools.ant.Main.startAnt(Main.java:236)
        at org.apache.tools.ant.launch.Launcher.run(Launcher.java:287)
        at org.apache.tools.ant.launch.Launcher.main(Launcher.java:113)

The memory is definitely enough. I incrased the bamboo JVM RAM from 512/1024 to 4096MB (both min/max). In the admin UI, there is always at least 50% free. The server itself has 24GB of memory.

It seems that the out of memory error doesn't indicate a lack of memory but lacking OS limits instead. In /etc/security/limits.conf I set the following limits (first to the user bamboo which is used for the jobs, later * for all users):

*      soft   nofile  16384
*      hard   nofile  16384
*      soft   nproc   65535
*      hard   nproc   65535

and also set this in /etc/sysctl.conf:

fs.file-max = 16384

To check that the limits works:

su - bamboo
ulimit -a
core file size          (blocks, -c) unlimited
data seg size           (kbytes, -d) unlimited
scheduling priority             (-e) 0
file size               (blocks, -f) unlimited
pending signals                 (-i) 94782
max locked memory       (kbytes, -l) 64
max memory size         (kbytes, -m) unlimited
open files                      (-n) 16384
pipe size            (512 bytes, -p) 8
POSIX message queues     (bytes, -q) 819200
real-time priority              (-r) 0
stack size              (kbytes, -s) 8192
cpu time               (seconds, -t) unlimited
max user processes              (-u) 65535
virtual memory          (kbytes, -v) unlimited
file locks                      (-x) unlimited

Since bamboo still crashes, I added a shell plan that executes

whoami
ulimit -a

with the following output:

build   07-Feb-2020 11:29:11    bamboo
build   07-Feb-2020 11:29:11    core file size          (blocks, -c) unlimited
build   07-Feb-2020 11:29:11    data seg size           (kbytes, -d) unlimited
build   07-Feb-2020 11:29:11    scheduling priority             (-e) 0
build   07-Feb-2020 11:29:11    file size               (blocks, -f) unlimited
build   07-Feb-2020 11:29:11    pending signals                 (-i) 94782
build   07-Feb-2020 11:29:11    max locked memory       (kbytes, -l) 64
build   07-Feb-2020 11:29:11    max memory size         (kbytes, -m) unlimited
build   07-Feb-2020 11:29:11    open files                      (-n) 4096
build   07-Feb-2020 11:29:11    pipe size            (512 bytes, -p) 8
build   07-Feb-2020 11:29:11    POSIX message queues     (bytes, -q) 819200
build   07-Feb-2020 11:29:11    real-time priority              (-r) 0
build   07-Feb-2020 11:29:11    stack size              (kbytes, -s) 8192
build   07-Feb-2020 11:29:11    cpu time               (seconds, -t) unlimited
build   07-Feb-2020 11:29:11    max user processes              (-u) 94782
build   07-Feb-2020 11:29:11    virtual memory          (kbytes, -v) unlimited
build   07-Feb-2020 11:29:11    file locks                      (-x) unlimited

We can see that e.g. open files ist just set to 4096 instead of 16384 as configured.

What is bamboo doing that he's ignoring those system-wide limits?

Lion
  • 496
  • 9
  • 19

1 Answers1

2

Problem was not caused by Bamboo this time, instead it seems that systemd ignores the system-wide limits. I had to edit the service:

vim /etc/systemd/system/bamboo.service

and manually add the limit in the [Service] section like this:

LimitNOFILE=65536

Keep in mind that in case of automatically generated services (e.g. by package manager) you should override it in /etc/systemd/system/<service-name>.service.d/override.conf so that the changes are presistent after package updates. On our server this was not important since the application is not avaliable during the package manager, so the service was created manually.

Lion
  • 496
  • 9
  • 19