NebulaGraph基于docker安装多机集群

# 创建目录，三台机器

mkdir -p /opt/NebulaGraph/{meta,graph,storage}/{bin,data,conf,logs}/

# 下载镜像,三台机器都下载

docker pull docker.io/vesoft/nebula-metad:nightly
docker pull docker.io/vesoft/nebula-graphd:nightly
docker pull docker.io/vesoft/nebula-storaged:nightly

vim  /opt/NebulaGraph/meta/conf/nebula-metad.conf

# 内容如下

########## basics ##########

# Whether to run as a daemon process

--daemonize=false

# The file to host the process id

--pid\_file=pids/nebula-metad.pid


########## logging ##########

# The directory to host logging files

--log\_dir=/logs

# Log level, 0, 1, 2, 3 for INFO, WARNING, ERROR, FATAL respectively

--minloglevel=0

# Verbose log level, 1, 2, 3, 4, the higher of the level, the more verbose of the logging

--v=0

# Maximum seconds to buffer the log messages

--logbufsecs=0

# Whether to redirect stdout and stderr to separate output files

--redirect\_stdout=true

# Destination filename of stdout and stderr, which will also reside in log\_dir.

--stdout\_log\_file=metad-stdout.log

--stderr\_log\_file=metad-stderr.log

# Copy log messages at or above this level to stderr in addition to logfiles. The numbers of severity levels INFO, WARNING, ERROR, and FATAL are 0, 1, 2, and 3, respectively.

--stderrthreshold=3

# wether logging files' name contain time stamp, If Using logrotate to rotate logging files, than should set it to true.

--timestamp\_in\_logfile\_name=true



########## networking ##########

# Comma separated Meta Server addresses

--meta\_server\_addrs=192.168.17.129:9559,192.168.17.130:9559,192.168.17.131:9559

# Local IP used to identify the nebula-metad process.

# Change it to an address other than loopback if the service is distributed or

# will be accessed remotely.

--local\_ip=192.168.17.129

# Meta daemon listening port

--port=9559

# HTTP service ip

--ws\_ip=0.0.0.0

# HTTP service port

--ws\_http\_port=19559

# Port to listen on Storage with HTTP protocol, it corresponds to ws\_http\_port in storage's configuration file

--ws\_storage\_http\_port=19779



########## storage ##########

# Root data path, here should be only single path for metad

--data\_path=/data/meta



########## Misc #########

# The default number of parts when a space is created

--default\_parts\_num=100

# The default replica factor when a space is created

--default\_replica\_factor=1



--heartbeat\_interval\_secs=10

--agent\_heartbeat\_interval\_secs=60

vim  /opt/NebulaGraph/graph/conf/nebula-graphd.conf



# 内容如下

########## basics ##########

# Whether to run as a daemon process

--daemonize=false

# The file to host the process id

--pid\_file=pids/nebula-graphd.pid

# Whether to enable optimizer

--enable\_optimizer=true

# The default charset when a space is created

--default\_charset=utf8

# The default collate when a space is created

--default\_collate=utf8\_bin

# Whether to use the configuration obtained from the configuration file

--local\_config=true



########## logging ##########

# The directory to host logging files

--log\_dir=/logs

# Log level, 0, 1, 2, 3 for INFO, WARNING, ERROR, FATAL respectively

--minloglevel=0

# Verbose log level, 1, 2, 3, 4, the higher of the level, the more verbose of the logging

--v=0

# Maximum seconds to buffer the log messages

--logbufsecs=0

# Whether to redirect stdout and stderr to separate output files

--redirect\_stdout=true

# Destination filename of stdout and stderr, which will also reside in log\_dir.

--stdout\_log\_file=graphd-stdout.log

--stderr\_log\_file=graphd-stderr.log

# Copy log messages at or above this level to stderr in addition to logfiles. The numbers of severity levels INFO, WARNING, ERROR, and FATAL are 0, 1, 2, and 3, respectively.

--stderrthreshold=3

# wether logging files' name contain time stamp.

--timestamp\_in\_logfile\_name=true

########## query ##########

# Whether to treat partial success as an error.

# This flag is only used for Read-only access, and Modify access always treats partial success as an error.

--accept\_partial\_success=false

# Maximum sentence length, unit byte

--max\_allowed\_query\_size=4194304



########## networking ##########

# Comma separated Meta Server Addresses

--meta\_server\_addrs=192.168.17.129:9559,192.168.17.130:9559,192.168.17.131:9559

# Local IP used to identify the nebula-graphd process.

# Change it to an address other than loopback if the service is distributed or

# will be accessed remotely.

--local\_ip=192.168.17.129

# Network device to listen on

--listen\_netdev=any

# Port to listen on

--port=9669

# To turn on SO\_REUSEPORT or not

--reuse\_port=false

# Backlog of the listen socket, adjust this together with net.core.somaxconn

--listen\_backlog=1024

# The number of seconds Nebula service waits before closing the idle connections

--client\_idle\_timeout\_secs=28800

# The number of seconds before idle sessions expire

# The range should be in [1, 604800]

--session\_idle\_timeout\_secs=28800

# The number of threads to accept incoming connections

--num\_accept\_threads=1

# The number of networking IO threads, 0 for # of CPU cores

--num\_netio\_threads=0

# Max active connections for all networking threads. 0 means no limit.

# Max connections for each networking thread = num\_max\_connections / num\_netio\_threads

--num\_max\_connections=0

# The number of threads to execute user queries, 0 for # of CPU cores

--num\_worker\_threads=0

# HTTP service ip

--ws\_ip=0.0.0.0

# HTTP service port

--ws\_http\_port=19669

# storage client timeout

--storage\_client\_timeout\_ms=60000

# slow query threshold in us

--slow\_query\_threshold\_us=200000

# Port to listen on Meta with HTTP protocol, it corresponds to ws\_http\_port in metad's configuration file

--ws\_meta\_http\_port=19559



########## authentication ##########

# Enable authorization

--enable\_authorize=false

# User login authentication type, password for nebula authentication, ldap for ldap authentication, cloud for cloud authentication

--auth\_type=password



########## memory ##########

# System memory high watermark ratio, cancel the memory checking when the ratio greater than 1.0

--system\_memory\_high\_watermark\_ratio=0.8



########## metrics ##########

--enable\_space\_level\_metrics=false



########## experimental feature ##########

# if use experimental features

--enable\_experimental\_feature=false



# if use balance data feature, only work if enable\_experimental\_feature is true

--enable\_data\_balance=true



# enable udf, written in c++ only for now

--enable\_udf=true



# set the directory where the .so files of udf are stored, when enable\_udf is true

--udf\_path=/home/nebula/dev/nebula/udf/



########## session ##########

# Maximum number of sessions that can be created per IP and per user

--max\_sessions\_per\_ip\_per\_user=300



########## memory tracker ##########

# trackable memory ratio (trackable\_memory / (total\_memory - untracked\_reserved\_memory) )

--memory\_tracker\_limit\_ratio=0.8

# untracked reserved memory in Mib

--memory\_tracker\_untracked\_reserved\_memory\_mb=50



# enable log memory tracker stats periodically

--memory\_tracker\_detail\_log=false

# log memory tacker stats interval in milliseconds

--memory\_tracker\_detail\_log\_interval\_ms=60000



# enable memory background purge (if jemalloc is used)

--memory\_purge\_enabled=true

# memory background purge interval in seconds

--memory\_purge\_interval\_seconds=10



########## performance optimization ##########

# The max job size in multi job mode

--max\_job\_size=1

# The min batch size for handling dataset in multi job mode, only enabled when max\_job\_size is greater than 1

--min\_batch\_size=8192

# if true, return directly without go through RPC

--optimize\_appendvertices=false

# number of paths constructed by each thread

--path\_batch\_size=10000

vim  /opt/NebulaGraph/storage/conf/nebula-storaged.conf



# 内容如下

########## basics ##########

# Whether to run as a daemon process

--daemonize=false

# The file to host the process id

--pid\_file=pids/nebula-storaged.pid

# Whether to use the configuration obtained from the configuration file

--local\_config=true



########## logging ##########

# The directory to host logging files

--log\_dir=/logs

# Log level, 0, 1, 2, 3 for INFO, WARNING, ERROR, FATAL respectively

--minloglevel=0

# Verbose log level, 1, 2, 3, 4, the higher of the level, the more verbose of the logging

--v=0

# Maximum seconds to buffer the log messages

--logbufsecs=0

# Whether to redirect stdout and stderr to separate output files

--redirect\_stdout=true

# Destination filename of stdout and stderr, which will also reside in log\_dir.

--stdout\_log\_file=storaged-stdout.log

--stderr\_log\_file=storaged-stderr.log

# Copy log messages at or above this level to stderr in addition to logfiles. The numbers of severity levels INFO, WARNING, ERROR, and FATAL are 0, 1, 2, and 3, respectively.

--stderrthreshold=3

# Wether logging files' name contain time stamp.

--timestamp\_in\_logfile\_name=true



########## networking ##########

# Comma separated Meta server addresses

--meta\_server\_addrs=192.168.17.129:9559,192.168.17.130:9559,192.168.17.131:9559

# Local IP used to identify the nebula-storaged process.

# Change it to an address other than loopback if the service is distributed or

# will be accessed remotely.

--local\_ip=192.168.17.129

# Storage daemon listening port

--port=9779

# HTTP service ip

--ws\_ip=0.0.0.0

# HTTP service port

--ws\_http\_port=19779

# heartbeat with meta service

--heartbeat\_interval\_secs=10



######### Raft #########

# Raft election timeout

--raft\_heartbeat\_interval\_secs=30

# RPC timeout for raft client (ms)

--raft\_rpc\_timeout\_ms=500

## recycle Raft WAL

--wal\_ttl=14400



########## Disk ##########

# Root data path. Split by comma. e.g. --data\_path=/disk1/path1/,/disk2/path2/

# One path per Rocksdb instance.

--data\_path=/data/storage



# Minimum reserved bytes of each data path

--minimum\_reserved\_bytes=268435456



# The default reserved bytes for one batch operation

--rocksdb\_batch\_size=4096

# The default block cache size used in BlockBasedTable.

# The unit is MB.

--rocksdb\_block\_cache=4

# The type of storage engine, `rocksdb', `memory', etc.

--engine\_type=rocksdb



# Compression algorithm, options: no,snappy,lz4,lz4hc,zlib,bzip2,zstd

# For the sake of binary compatibility, the default value is snappy.

# Recommend to use:

#   \* lz4 to gain more CPU performance, with the same compression ratio with snappy

#   \* zstd to occupy less disk space

#   \* lz4hc for the read-heavy write-light scenario

--rocksdb\_compression=lz4



# Set different compressions for different levels

# For example, if --rocksdb\_compression is snappy,

# "no:no:lz4:lz4::zstd" is identical to "no:no:lz4:lz4:snappy:zstd:snappy"

# In order to disable compression for level 0/1, set it to "no:no"

--rocksdb\_compression\_per\_level=



# Whether or not to enable rocksdb's statistics, disabled by default

--enable\_rocksdb\_statistics=false



# Statslevel used by rocksdb to collection statistics, optional values are

#   \* kExceptHistogramOrTimers, disable timer stats, and skip histogram stats

#   \* kExceptTimers, Skip timer stats

#   \* kExceptDetailedTimers, Collect all stats except time inside mutex lock AND time spent on compression.

#   \* kExceptTimeForMutex, Collect all stats except the counters requiring to get time inside the mutex lock.

#   \* kAll, Collect all stats

--rocksdb\_stats\_level=kExceptHistogramOrTimers



# Whether or not to enable rocksdb's prefix bloom filter, enabled by default.

--enable\_rocksdb\_prefix\_filtering=true

# Whether or not to enable rocksdb's whole key bloom filter, disabled by default.

--enable\_rocksdb\_whole\_key\_filtering=false



############## rocksdb Options ##############

# rocksdb DBOptions in json, each name and value of option is a string, given as "option\_name":"option\_value" separated by comma

--rocksdb\_db\_options={}

# rocksdb ColumnFamilyOptions in json, each name and value of option is string, given as "option\_name":"option\_value" separated by comma

--rocksdb\_column\_family\_options={"write\_buffer\_size":"67108864","max\_write\_buffer\_number":"4","max\_bytes\_for\_level\_base":"268435456"}

# rocksdb BlockBasedTableOptions in json, each name and value of option is string, given as "option\_name":"option\_value" separated by comma

--rocksdb\_block\_based\_table\_options={"block\_size":"8192"}



############### misc ####################

# Whether turn on query in multiple thread

--query\_concurrently=true

# Whether remove outdated space data

--auto\_remove\_invalid\_space=true

# Network IO threads number

--num\_io\_threads=16

# Max active connections for all networking threads. 0 means no limit.

# Max connections for each networking thread = num\_max\_connections / num\_netio\_threads

--num\_max\_connections=0

# Worker threads number to handle request

--num\_worker\_threads=32

# Maximum subtasks to run admin jobs concurrently

--max\_concurrent\_subtasks=10

# The rate limit in bytes when leader synchronizes snapshot data

--snapshot\_part\_rate\_limit=10485760

# The amount of data sent in each batch when leader synchronizes snapshot data

--snapshot\_batch\_size=1048576

# The rate limit in bytes when leader synchronizes rebuilding index

--rebuild\_index\_part\_rate\_limit=4194304

# The amount of data sent in each batch when leader synchronizes rebuilding index

--rebuild\_index\_batch\_size=1048576



########## memory tracker ##########

# trackable memory ratio (trackable\_memory / (total\_memory - untracked\_reserved\_memory) )

--memory\_tracker\_limit\_ratio=0.8

# untracked reserved memory in Mib

--memory\_tracker\_untracked\_reserved\_memory\_mb=50



# enable log memory tracker stats periodically

--memory\_tracker\_detail\_log=false

# log memory tacker stats interval in milliseconds

--memory\_tracker\_detail\_log\_interval\_ms=60000



# enable memory background purge (if jemalloc is used)

--memory\_purge\_enabled=true

# memory background purge interval in seconds

--memory\_purge\_interval\_seconds=10

vim  /opt/NebulaGraph/meta/bin/start.sh



# 内容 注意不同节点修改hostname、name

#!/bin/bash

docker run -d --network=host --hostname=ubuntu01 \

        --name meta01 --restart=always\

        -e USER=root \

        -e TZ=Asia/Shanghai \

        --cap-add SYS\_PTRACE \

        -v  /opt/NebulaGraph/meta/conf/nebula-metad.conf:/usr/local/nebula/etc/nebula-metad.conf \

        -v  /opt/NebulaGraph/meta/data:/data \

        -v  /opt/NebulaGraph/meta/logs:/logs \

        docker.io/vesoft/nebula-metad:nightly



# 修改权限

chmod 755 /opt/NebulaGraph/meta/bin/start.sh

vim  /opt/NebulaGraph/graph/bin/start.sh



# 内容 注意不同节点修改hostname、name

#!/bin/bash

docker run -d --network=host --hostname=ubuntu01 \

        --name graph01 --restart=always\

        -e USER=root \

        -e TZ=Asia/Shanghai \

        --cap-add SYS\_PTRACE \

        -v  /opt/NebulaGraph/graph/conf/nebula-graphd.conf:/usr/local/nebula/etc/nebula-graphd.conf \

        -v  /opt/NebulaGraph/graph/logs:/logs \

        docker.io/vesoft/nebula-graphd:nightly



# 修改权限

chmod 755 /opt/NebulaGraph/graph/bin/start.sh

vim /opt/NebulaGraph/storage/bin/start.sh



# 内容 注意不同节点修改hostname、name

#!/bin/bash

docker run -d --network=host --hostname=ubuntu01 \

        --name storage01 --restart=always\

        -e USER=root \

        -e TZ=Asia/Shanghai \

        --cap-add SYS\_PTRACE \

        -v  /opt/NebulaGraph/storage/conf/nebula-storaged.conf:/usr/local/nebula/etc/nebula-storaged.conf \

        -v  /opt/NebulaGraph/storage/data:/data \

        -v  /opt/NebulaGraph/storage/logs:/logs \

        docker.io/vesoft/nebula-storaged:nightly




# 修改权限

chmod 755 /opt/NebulaGraph/storage/bin/start.sh

# 启动应用



/opt/NebulaGraph/meta/bin/start.sh

/opt/NebulaGraph/storage/bin/start.sh

/opt/NebulaGraph/graph/bin/start.sh

# 连接

nebula-console -addr 192.168.17.129 --port 9669 -u root -p password


# 注册storage

ADD HOSTS 192.168.17.129:9779,192.168.17.130:9779,192.168.17.131:9779

# 连接
nebula-console -addr 192.168.17.129 --port 9669 -u root -p password

# 查看meta
show hosts meta

# 查看graph
show hosts graph

# 查看storage
show hosts