Skip to content

Commit 75d0a3f

Browse files
author
arnett, stu
committed
v2.1.1
1 parent ab441d9 commit 75d0a3f

35 files changed

+1583
-162
lines changed

‎.gitattributes

+1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
* text=auto
22

33
*.bat text eol=crlf
4+
*.sh text eol=lf

‎build.gradle

+2-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
*/
1515
allprojects {
1616
group = 'com.emc.ecs'
17-
version = '2.1'
17+
version = '2.1.1'
1818
}
1919

2020
ext.mainClass = 'com.emc.ecs.sync.EcsSync'
@@ -117,6 +117,7 @@ distributions {
117117
from 'readme.txt'
118118
from 'license.txt'
119119
from shadowJar
120+
from { project(':ecs-sync-ctl').shadowJar }
120121
from 'script'
121122
into('sample') {
122123
from 'sample'

‎ecs-sync-ui/build.gradle

-3
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,6 @@ plugins {
1818
id "io.spring.dependency-management" version "0.5.2.RELEASE"
1919
}
2020

21-
version "2.1"
22-
group "com.emc.ecs"
23-
2421
apply plugin: "spring-boot"
2522
apply plugin: "war"
2623
apply plugin: "asset-pipeline"

‎ecs-sync-ui/grails-app/assets/javascripts/application.js

+13
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,19 @@ if (typeof jQuery !== 'undefined') {
2222

2323
$(document).ready(function() {
2424
$('.disabled').prop('disabled', true);
25+
26+
$('.passwordToggle').click(function() {
27+
var $this = $(this);
28+
var $target = $('#' + $this.data('targetId'));
29+
var isPassword = $target.attr('type') == 'password';
30+
if (isPassword) {
31+
$target[0].type = 'text';
32+
$this.text('hide');
33+
} else {
34+
$target[0].type = 'password';
35+
$this.text('show');
36+
}
37+
});
2538
});
2639

2740
var showAdvanced = false;

‎ecs-sync-ui/grails-app/assets/stylesheets/application.css

+5
Original file line numberDiff line numberDiff line change
@@ -56,3 +56,8 @@ body {
5656
.kv-table th:first-child {
5757
width: 200px;
5858
}
59+
60+
.passwordToggle {
61+
text-decoration: underline;
62+
cursor: pointer;
63+
}

‎ecs-sync-ui/grails-app/services/sync/ui/ReportService.groovy

+5-8
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,6 @@ class ReportService {
1919
def targetPath = SyncUtil.getTargetPath(sync)
2020
Date startTime = new Date(progress.syncStartTime)
2121

22-
def modifiedSince = sync.source.customProperties.find { it.key == 'modifiedSince' }?.value ?: 'N/A'
23-
2422
// generate summary CSV
2523
long xputBytes = 0
2624
if (progress.bytesComplete && progress.runtimeMs)
@@ -33,22 +31,21 @@ class ReportService {
3331
cpuUsage = (double) progress.cpuTimeMs / (double) progress.runtimeMs / (double) host.hostCpuCount * 100
3432
def rows = [[]]
3533
rows << ['Source', "${sync.source.pluginClass.tokenize('.').last()}"]
34+
rows << ['Source Path', "${sourcePath}"]
3635
rows << ['Target', "${sync.target.pluginClass.tokenize('.').last()}"]
36+
rows << ['Target Path', "${targetPath}"]
3737
rows << ['Started At', "${startTime.format('yyyy-MM-dd hh:mm:ssa')}"]
3838
rows << ['Stopped At', "${progress.syncStopTime ? new Date(progress.syncStopTime).format('yyyy-MM-dd hh:mm:ssa') : 'N/A'}"]
3939
rows << ['Duration', "${DisplayUtil.shortDur(TimeCategory.minus(new Date(progress.runtimeMs), new Date(0)))}"]
40-
rows << ['Source Path', "${sourcePath}"]
41-
rows << ['Modified Since', "${modifiedSince}"]
42-
rows << ['Target Path', "${targetPath}"]
4340
rows << ['Thread Count', "${sync.syncThreadCount}"]
44-
rows << ['Total Files and Directories Processed', "${progress.objectsComplete}"]
4541
rows << ['Total Files and Directories Estimated', "${progress.totalObjectsExpected}"]
46-
rows << ['Total Bytes Transferred', "${DisplayUtil.simpleSize(progress.bytesComplete)}B"]
42+
rows << ['Total Files and Directories Processed', "${progress.objectsComplete}"]
4743
rows << ['Total Bytes Estimated', "${progress.totalBytesExpected ? DisplayUtil.simpleSize(progress.totalBytesExpected) + 'B' : 'N/A'}"]
44+
rows << ['Total Bytes Transferred', "${DisplayUtil.simpleSize(progress.bytesComplete)}B"]
4845
rows << ['General Error Message', "${progress.runError}"]
4946
rows << ['Total Errors', "${progress.objectsFailed}"]
50-
rows << ['Overall Throughput (bytes)', "${DisplayUtil.simpleSize(xputBytes)}B/s"]
5147
rows << ['Overall Throughput (files)', "${DisplayUtil.simpleSize(xputFiles)}/s"]
48+
rows << ['Overall Throughput (bytes)', "${DisplayUtil.simpleSize(xputBytes)}B/s"]
5249
rows << ['Overall CPU Usage', "${cpuUsage.trunc(1)}%"]
5350

5451
def entry = new ResultEntry([ecsService: ecsService, jobId: jobId.toLong(), startTime: startTime])

‎ecs-sync-ui/grails-app/views/uiConfig/create.gsp

+2-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,8 @@
3030
${it.radio} ${it.label} &nbsp;&nbsp;&nbsp;&nbsp;</g:radioGroup></td></tr>
3131
<tr class="advanced"><th>Port: </th><td><g:textField name="port" value="${uiConfig.port}" size="40" /></td></tr>
3232
<tr><th>ECS User: </th><td><g:textField name="accessKey" value="${uiConfig.accessKey}" size="40" /></td></tr>
33-
<tr><th>ECS Secret: </th><td><g:textField name="secretKey" value="${uiConfig.secretKey}" size="60" /></td></tr>
33+
<tr><th>ECS Secret: </th><td><g:passwordField name="secretKey" value="${uiConfig.secretKey}" size="60" />
34+
<span class="passwordToggle" data-target-id="secretKey">show</span></td></tr>
3435
<tr class="advanced"><th>Config Bucket: </th><td><g:textField name="configBucket" value="${uiConfig.configBucket}" /></td></tr>
3536
<tr><td><g:submitButton name="readEcs" value="Read Configuration from ECS" class="btn btn-primary" /></td>
3637
<td><a href="#" onclick="toggleAdvanced($(this))">show advanced options</a></td></tr>

‎ecs-sync-ui/grails-app/views/uiConfig/edit.gsp

+2-1
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,8 @@
3333
${it.radio} ${it.label} &nbsp;&nbsp;&nbsp;&nbsp;</g:radioGroup></td></tr>
3434
<tr class="advanced"><th>Port: </th><td><g:textField name="port" value="${uiConfig.port}" size="40" /></td></tr>
3535
<tr><th>ECS User: </th><td><g:textField name="accessKey" value="${uiConfig.accessKey}" size="40" /></td></tr>
36-
<tr><th>ECS Secret: </th><td><g:textField name="secretKey" value="${uiConfig.secretKey}" size="60" /></td></tr>
36+
<tr><th>ECS Secret: </th><td><g:passwordField name="secretKey" value="${uiConfig.secretKey}" size="60" />
37+
<span class="passwordToggle" data-target-id="secretKey">show</span></td></tr>
3738
<tr class="advanced"><th>Config Bucket: </th><td><g:textField name="configBucket" value="${uiConfig.configBucket}" /></td></tr>
3839
<tr><td><g:submitButton name="readEcs" value="Read Configuration from ECS" class="btn btn-primary" /></td>
3940
<td><a href="#" onclick="toggleAdvanced($(this))">show advanced options</a></td></tr>

‎ecs-sync-ui/src/main/groovy/sync/ui/SyncUtil.groovy

+5-4
Original file line numberDiff line numberDiff line change
@@ -8,31 +8,31 @@ class SyncUtil {
88
def sourcePath = syncConfig.Source.Property.find { it.@name == 'rootFile' }?.@value?.toString()
99
if (!sourcePath)
1010
sourcePath = syncConfig.Source.Property.find { it.@name == 'bucketName' }?.@value?.toString() + '/' +
11-
syncConfig.Source.Property.find { it.@name == 'rootKey' }?.@value?.toString()
11+
syncConfig.Source.Property.find { it.@name == 'rootKey' }?.@value?.toString() ?: ''
1212
return sourcePath
1313
}
1414

1515
static String getSourcePath(SyncConfig syncConfig) {
1616
def sourcePath = syncConfig.source.customProperties.find { it.key == 'rootFile' }?.value
1717
if (!sourcePath)
1818
sourcePath = syncConfig.source.customProperties.find { it.key == 'bucketName' }?.value + '/' +
19-
syncConfig.source.customProperties.find { it.key == 'rootKey' }?.value
19+
syncConfig.source.customProperties.find { it.key == 'rootKey' }?.value ?: ''
2020
return sourcePath
2121
}
2222

2323
static String getTargetPath(GPathResult syncConfig) {
2424
def targetPath = syncConfig.Target.Property.find { it.@name == 'targetRoot' }?.@value?.toString()
2525
if (!targetPath)
2626
targetPath = syncConfig.Target.Property.find { it.@name == 'bucketName' }?.@value?.toString() + '/' +
27-
syncConfig.Target.Property.find { it.@name == 'rootKey' }?.@value?.toString()
27+
syncConfig.Target.Property.find { it.@name == 'rootKey' }?.@value?.toString() ?: ''
2828
return targetPath
2929
}
3030

3131
static String getTargetPath(SyncConfig syncConfig) {
3232
def targetPath = syncConfig.target.customProperties.find { it.key == 'targetRoot' }?.value
3333
if (!targetPath)
3434
targetPath = syncConfig.target.customProperties.find { it.key == 'bucketName' }?.value + '/' +
35-
syncConfig.target.customProperties.find { it.key == 'rootKey' }?.value
35+
syncConfig.target.customProperties.find { it.key == 'rootKey' }?.value ?: ''
3636
return targetPath
3737
}
3838

@@ -46,6 +46,7 @@ class SyncUtil {
4646
}
4747

4848
static configureDatabase(syncConfig, dbType, dbName, grailsApplication) {
49+
dbName = dbName.replaceAll(/[^_0-9a-zA-Z]/, '_')
4950
if (dbType && dbType != 'None') {
5051
if (dbType == 'Sqlite') {
5152
def dbDir = grailsApplication.config.sync.dbDir

‎script/ova/README

+32
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,25 @@
1+
Standard Install Layout
2+
=========
3+
4+
/opt/emc/ecs-sync this directory should be created and owned by the ecssync user. contains all .jar files with
5+
symbolic links to specific versions (i.e. ecs-sync.jar -> ecs-sync-2.1.jar). also contains the
6+
application-production.yml config file
7+
8+
/var/log/ecs-sync log directory. this should be created and owned by the ecssync user.
9+
10+
/etc/init.d service script location. place scripts from init.d/ (in distribution zip) here
11+
12+
/etc/logrotate.d log rotate scripts. place scripts from logrotate.d/ (in distribution zip) here
13+
14+
install.sh convenience installation script; basically does all of the above
15+
16+
/etc/httpd apache config dir. place files from httpd/ (in distribution zip) here to access the UI
17+
18+
configure-centos.sh configuration script for CentOS; installs most deps and configures apache, mysql, etc.
19+
and also takes care of the extra configuration steps below
20+
21+
=========
22+
123
NOTE: if you find the services are only listening on ipv6 localhost (:::1), you need to disable ipv6 in the OS.
224
in /etc/sysctl.conf:
325

@@ -8,3 +30,13 @@ Also, if you're having trouble with apache and ProxyPass (disabled connection fr
830
to connect to the network:
931

1032
setsebool -P httpd_can_network_connect=1
33+
34+
Also a good idea to reduce "swappiness" of virtual memory since these processes use lots of RAM. 10 is recommended
35+
36+
in /etc/sysctl.conf:
37+
38+
vm.swappiness = 10
39+
40+
then:
41+
42+
sysctl -p

‎script/ova/configure-centos.sh

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
#!/bin/bash
2+
if [ ! -f /etc/redhat-release ]; then
3+
echo "This script is designed for RedHat-based systems"
4+
exit 1
5+
fi
6+
if [ "$(id -u)" != "0" ]; then
7+
echo "Please run as root"
8+
exit 1
9+
fi
10+
DIST_DIR="$(cd "$(dirname $0)/.." && pwd)"
11+
echo "DIST_DIR=${DIST_DIR}"
12+
13+
# set hostname
14+
hostname ecssync.local
15+
16+
# EPEL repo
17+
yum -y install epel-release
18+
19+
# java
20+
yum -y install java
21+
22+
# NFS/SMB client tools
23+
yum -y install nfs-utils nfs-utils-lib samba-client cifs-utils
24+
25+
# analysis tools
26+
yum -y install iperf telnet
27+
28+
# apache
29+
yum -y install httpd mod_ssl
30+
# configure proxy and auth
31+
cp "${DIST_DIR}/ova/httpd/.htaccess" /etc/httpd
32+
cp "${DIST_DIR}/ova/httpd/conf.d/ecs-sync.conf" /etc/httpd/conf.d
33+
systemctl enable httpd
34+
systemctl start httpd
35+
# allow apache to use the network
36+
setsebool -P httpd_can_network_connect=1
37+
38+
# mysql (mariadb)
39+
yum -y install mariadb-server
40+
# enable UTF-8 support
41+
if [ -f /etc/my.cnf.d/server.cnf ]; then
42+
sed -i '/\[server\]/a\
43+
innodb_file_format=Barracuda\
44+
innodb_large_prefix=1\
45+
innodb_file_per_table=1' /etc/my.cnf.d/server.cnf
46+
fi
47+
systemctl daemon-reload
48+
systemctl enable mariadb-server
49+
systemctl start mariadb-server
50+
# remove test DBs and set root PW
51+
mysql_secure_installation
52+
# create database for ecs-sync
53+
MYSQL_DIR="$(cd "$(dirname $0)/../mysql" && pwd)"
54+
mysql -u root -p < "${MYSQL_DIR}/utf8/create_mysql_user_db.sql"
55+
56+
# sysctl tweaks
57+
echo '
58+
net.ipv6.conf.all.disable_ipv6 = 1
59+
net.ipv6.conf.default.disable_ipv6 = 1
60+
61+
vm.swappiness = 10' >> /etc/sysctl.conf
62+
sysctl -p
63+
64+
# configure LD_LIBRARY_PATH for CAS SDK
65+
echo '
66+
export LD_LIBRARY_PATH=/usr/local/Centera_SDK/lib/64' >> ~root/.bash_profile
67+
68+
echo 'Done (please manually install iozone, bucket-perf and the CAS SDK)'

‎script/ova/httpd/.htaccess

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
admin:$apr1$nXzDaB7N$T899OmZQ4nOrxEDlCFb4S1

‎script/ova/httpd/conf.d/ecs-sync.conf

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# Proxy config for sync-ui
2+
<Location />
3+
SSLRequireSSL
4+
5+
AuthType basic
6+
AuthName "ECS Sync"
7+
AuthUserFile .htpasswd
8+
Require valid-user
9+
10+
ProxyPass http://localhost:8080/
11+
ProxyPassReverse http://localhost:8080/
12+
</Location>

‎script/ova/init.d/ecs-sync

+33-21
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
1-
#!/bin/sh
1+
#!/bin/bash
22
#
33
# /etc/init.d/ecs-sync
44
# Subsystem file for "ecs-sync" service.
55
#
6-
# chkconfig: 3 50 05
6+
# chkconfig: 345 90 10
77
# description: ECS-Sync 2.0+ server deamon.
88
#
99
# processname: ecs-sync
@@ -14,35 +14,47 @@ SERVICE_NAME=ecs-sync
1414
INSTALL_DIR=/opt/emc/ecs-sync
1515
LOG_DIR=/var/log/ecs-sync
1616

17-
PATH_TO_JAR=$INSTALL_DIR/ecs-sync.jar
18-
PID_PATH_NAME=$LOG_DIR/ecs-sync.pid
19-
PATH_TO_LOG=$LOG_DIR/ecs-sync.log
17+
PATH_TO_JAR=$INSTALL_DIR/$SERVICE_NAME.jar
18+
PID_PATH_NAME=$LOG_DIR/$SERVICE_NAME.pid
19+
PATH_TO_LOG=$LOG_DIR/$SERVICE_NAME.log
2020
APP_USER=root
21-
JAVA_OPTS=-Xmx8G
21+
JAVA_OPTS="-server -Xshare:off -Xmx8G -XX:+UseParallelGC -XX:+EliminateLocks -XX:+UseBiasedLocking"
22+
23+
RUN_DIR=${INSTALL_DIR}
24+
CMD="nohup java ${JAVA_OPTS} -jar ${PATH_TO_JAR} --rest-only"
25+
26+
if [ "$(id -u)" != "0" ]; then
27+
echo "Please run as root"
28+
exit 1
29+
fi
2230

2331
start() {
24-
echo "Starting $SERVICE_NAME ..."
25-
if [ ! -f $PID_PATH_NAME ]; then
26-
su - $APP_USER <<EOF
27-
cd $INSTALL_DIR
28-
nohup java $JAVA_OPTS -jar $PATH_TO_JAR --rest-only >> $PATH_TO_LOG 2>&1 &
29-
echo \$! > $PID_PATH_NAME
32+
echo "Starting ${SERVICE_NAME} ..."
33+
if -f ${PID_PATH_NAME} -a ! ps -p $(cat ${PID_PATH_NAME}) > /dev/null 2>&1; then
34+
echo "Removing stale pidfile ..."
35+
rm ${PID_PATH_NAME}
36+
fi
37+
if [ ! -f ${PID_PATH_NAME} ]; then
38+
su - ${APP_USER} <<EOF
39+
cd ${RUN_DIR}
40+
${CMD} >> ${PATH_TO_LOG} 2>&1 &
41+
echo \$! > ${PID_PATH_NAME}
3042
EOF
31-
echo "$SERVICE_NAME started ..."
43+
echo "${SERVICE_NAME} started ..."
3244
else
33-
echo "$SERVICE_NAME is already running ..."
45+
echo "${SERVICE_NAME} is already running ..."
3446
fi
3547
}
3648

3749
stop() {
38-
if [ -f $PID_PATH_NAME ]; then
39-
PID=$(cat $PID_PATH_NAME);
40-
echo "$SERVICE_NAME stopping ..."
41-
kill -9 $PID;
42-
echo "$SERVICE_NAME stopped ..."
43-
rm $PID_PATH_NAME
50+
if [ -f ${PID_PATH_NAME} ]; then
51+
PID=$(cat ${PID_PATH_NAME})
52+
echo "${SERVICE_NAME} stopping ..."
53+
kill -9 ${PID}
54+
echo "${SERVICE_NAME} stopped ..."
55+
rm ${PID_PATH_NAME}
4456
else
45-
echo "$SERVICE_NAME is not running ..."
57+
echo "${SERVICE_NAME} is not running ..."
4658
fi
4759
}
4860

0 commit comments

Comments
 (0)