mirror of
https://github.com/NationalSecurityAgency/timely.git
synced 2026-11-10 03:41:25 -10:00
Compare commits
419
Commits
0.0.5-docs
..
2.1.0
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8db5358b50 | ||
|
|
3e376e106b | ||
|
|
18eecfd7b0 | ||
|
|
065d8a9a9f | ||
|
|
09c2ef7e11 | ||
|
|
cb875819ff | ||
|
|
fcc5f0f716 | ||
|
|
c87543f7d7 | ||
|
|
d7f5a3b20b | ||
|
|
9601cdc927 | ||
|
|
60c2588c79 | ||
|
|
f3275c6a34 | ||
|
|
7486b6b761 | ||
|
|
b1a968590c | ||
|
|
8f86b20c02 | ||
|
|
a3620a3f24 | ||
|
|
9c99e74958 | ||
|
|
91ec36e819 | ||
|
|
9ad1b18b74 | ||
|
|
82c45d4dd4 | ||
|
|
a0e47d0b48 | ||
|
|
d3dc324d84 | ||
|
|
4871b3a4a9 | ||
|
|
8d5d5eeca8 | ||
|
|
a006a44372 | ||
|
|
6cfcdc1030 | ||
|
|
4eb7ceff8c | ||
|
|
6ad8da2b99 | ||
|
|
7f56a36906 | ||
|
|
1443fdceb9 | ||
|
|
3cff14eb03 | ||
|
|
8fa5b56e62 | ||
|
|
b2f2cef834 | ||
|
|
c6a2988083 | ||
|
|
d54817cc9a | ||
|
|
c4ee035584 | ||
|
|
d4bfc1a3aa | ||
|
|
fb315148a1 | ||
|
|
e8ae575086 | ||
|
|
da0146c741 | ||
|
|
882b4e2587 | ||
|
|
8048862f2c | ||
|
|
c2f00c3d19 | ||
|
|
61b1a245e2 | ||
|
|
a3c33115c4 | ||
|
|
bed1ceb9ee | ||
|
|
06e2699a44 | ||
|
|
d09faaa087 | ||
|
|
088f1a5cab | ||
|
|
cf6d4ada3e | ||
|
|
e464a8c9e0 | ||
|
|
8e4e61c373 | ||
|
|
729dca23ba | ||
|
|
104d20ad1b | ||
|
|
f0f005f8fa | ||
|
|
8593218b89 | ||
|
|
cbfe6a439f | ||
|
|
455616d568 | ||
|
|
9d3172cc75 | ||
|
|
7aa7b0da2a | ||
|
|
6c61861d83 | ||
|
|
91882ba363 | ||
|
|
ed61b5f305 | ||
|
|
a083c2d632 | ||
|
|
2891794174 | ||
|
|
c99b2471cf | ||
|
|
bb479a58d7 | ||
|
|
ff37d40801 | ||
|
|
a7d980975e | ||
|
|
30f5a0c1b2 | ||
|
|
2f9e6ca2d7 | ||
|
|
954528a460 | ||
|
|
0f7b514b30 | ||
|
|
a6ca46e1c1 | ||
|
|
487d1d4a5c | ||
|
|
e3c26f815c | ||
|
|
bbc1946204 | ||
|
|
991412f99b | ||
|
|
5428e93cb5 | ||
|
|
c18aae5a89 | ||
|
|
6972c46a1a | ||
|
|
4d7bee7566 | ||
|
|
4b11b3d50f | ||
|
|
84b9856ec7 | ||
|
|
c9548a9b9a | ||
|
|
aec7bd4c38 | ||
|
|
e2b8b643f2 | ||
|
|
c19e1d8b2f | ||
|
|
cb73e8e75c | ||
|
|
4c508c1a7d | ||
|
|
c6c2c0d41e | ||
|
|
a24cef78b3 | ||
|
|
08f99f6540 | ||
|
|
be11da7d15 | ||
|
|
4bc5753328 | ||
|
|
2ccb89a06d | ||
|
|
dfc3840ad4 | ||
|
|
6235034d8b | ||
|
|
d8de6ac06f | ||
|
|
f381946abe | ||
|
|
48bbe6a561 | ||
|
|
0f6110ea59 | ||
|
|
461b5e7e78 | ||
|
|
4ec7d94c28 | ||
|
|
77e910ae06 | ||
|
|
a4998af887 | ||
|
|
2858549ede | ||
|
|
57f0e1408f | ||
|
|
151adc025a | ||
|
|
5abd9d7200 | ||
|
|
fa2e211c72 | ||
|
|
a21438976e | ||
|
|
ba37047f3a | ||
|
|
9305470c69 | ||
|
|
200233542e | ||
|
|
3cd78dc624 | ||
|
|
9027cac6e3 | ||
|
|
3663b4e6d5 | ||
|
|
64973d0674 | ||
|
|
5512306f1e | ||
|
|
d35ae08188 | ||
|
|
e448657afb | ||
|
|
1dcd893711 | ||
|
|
409f5375d1 | ||
|
|
c5df8139ca | ||
|
|
dcd23d4fc8 | ||
|
|
cc7326fc19 | ||
|
|
0d2b6e61c8 | ||
|
|
b7cd8765c5 | ||
|
|
a9d27d5b1b | ||
|
|
7d67eb218d | ||
|
|
5fb7e3dd2b | ||
|
|
5a89861cec | ||
|
|
a3bc9a01ee | ||
|
|
dba494b06d | ||
|
|
5826884c3c | ||
|
|
f94c731bbc | ||
|
|
4e042bcb94 | ||
|
|
ab7f499134 | ||
|
|
c098f9f3e4 | ||
|
|
94cfcbc0c4 | ||
|
|
674e1b0ab5 | ||
|
|
4eedb84547 | ||
|
|
bef86eac37 | ||
|
|
27bb16e403 | ||
|
|
e2bbe4e212 | ||
|
|
a95d2d7ea1 | ||
|
|
f751175190 | ||
|
|
648199f330 | ||
|
|
fcfc3f7bc0 | ||
|
|
200bc39902 | ||
|
|
f854c9e394 | ||
|
|
73ddae1f8d | ||
|
|
cb7184503e | ||
|
|
3a357b50ef | ||
|
|
689aee0678 | ||
|
|
32f6b9d4c9 | ||
|
|
512f318387 | ||
|
|
e04cfb310b | ||
|
|
0cecc3b191 | ||
|
|
b3d75e4c68 | ||
|
|
004c344153 | ||
|
|
2590effeee | ||
|
|
f49c83601a | ||
|
|
df035259c9 | ||
|
|
3a8cbd3378 | ||
|
|
39bea1da8d | ||
|
|
1d3f8b7d3a | ||
|
|
0fbbb86768 | ||
|
|
8b807e5ac7 | ||
|
|
710c789984 | ||
|
|
cf0d63a4e7 | ||
|
|
bfc67db185 | ||
|
|
f604314640 | ||
|
|
25c9e93888 | ||
|
|
a323d198e9 | ||
|
|
8e6cc4e0b4 | ||
|
|
3a3d98ec81 | ||
|
|
7599a2ac60 | ||
|
|
3d33ba2e31 | ||
|
|
bbbb453818 | ||
|
|
54951c05e6 | ||
|
|
b63914e02c | ||
|
|
1f386db00a | ||
|
|
49222ac76a | ||
|
|
38f2c6cdfa | ||
|
|
3926e6f066 | ||
|
|
30800a0b5c | ||
|
|
92b23a4c51 | ||
|
|
8030ea59a5 | ||
|
|
a4c8158003 | ||
|
|
acba454ee3 | ||
|
|
30790608ca | ||
|
|
f2f3788f99 | ||
|
|
326b9dc8f1 | ||
|
|
3f3b605c63 | ||
|
|
5580a2fa66 | ||
|
|
1a54c3ada3 | ||
|
|
2080764e0d | ||
|
|
e7dade04ee | ||
|
|
7ed337483e | ||
|
|
b070bc3c47 | ||
|
|
51f9103741 | ||
|
|
a97fa83e2f | ||
|
|
0b5c3544db | ||
|
|
186b60756a | ||
|
|
133a6bf5cf | ||
|
|
b684d7eefe | ||
|
|
73f1603f1e | ||
|
|
9267e0c915 | ||
|
|
1ce49628e5 | ||
|
|
acad3af083 | ||
|
|
b4635f1a55 | ||
|
|
4c1ede04df | ||
|
|
2841b0a3e9 | ||
|
|
6efa13909f | ||
|
|
368c572c67 | ||
|
|
554baf6af7 | ||
|
|
1ef11799be | ||
|
|
16b664297d | ||
|
|
4277b63bc3 | ||
|
|
5a9ad3cb27 | ||
|
|
f990e89ab3 | ||
|
|
c01ac80f23 | ||
|
|
35e8f99d09 | ||
|
|
847144d236 | ||
|
|
b3bfbc1e1f | ||
|
|
8d86e3894a | ||
|
|
99b9181902 | ||
|
|
3c0bf87b79 | ||
|
|
d2ba7f635e | ||
|
|
92510fdc83 | ||
|
|
4eac17fbb5 | ||
|
|
328ae075f4 | ||
|
|
cf12295ac8 | ||
|
|
df180504f2 | ||
|
|
021ff832ef | ||
|
|
6fc9066e70 | ||
|
|
f11b82b662 | ||
|
|
dcfd165a9b | ||
|
|
fd98558218 | ||
|
|
f47cd66ec2 | ||
|
|
ec4ba6e7f5 | ||
|
|
e698734e4b | ||
|
|
78b49fe44e | ||
|
|
f109f04162 | ||
|
|
4bf68fe328 | ||
|
|
9eb8da8e80 | ||
|
|
f54705374e | ||
|
|
5e166f0044 | ||
|
|
17527af6b4 | ||
|
|
64f7890f8b | ||
|
|
e7c058ef56 | ||
|
|
6415d100fc | ||
|
|
046000c16a | ||
|
|
d0f6534296 | ||
|
|
07c04425e1 | ||
|
|
113b6464f3 | ||
|
|
e2eda4c74a | ||
|
|
68b8d6cd51 | ||
|
|
458028d5df | ||
|
|
320f291785 | ||
|
|
95c2654357 | ||
|
|
045964c01d | ||
|
|
2f8759e32b | ||
|
|
451f001b9f | ||
|
|
044f6576fd | ||
|
|
520edee16f | ||
|
|
8034a673fc | ||
|
|
eb9e874a57 | ||
|
|
1658312512 | ||
|
|
2fdfed7554 | ||
|
|
e90340be09 | ||
|
|
9770dce1cd | ||
|
|
8f98199a37 | ||
|
|
425616d536 | ||
|
|
158ee4fd55 | ||
|
|
89e8fd7ebd | ||
|
|
cb5794491a | ||
|
|
8b4f0d5764 | ||
|
|
c90b86c4c9 | ||
|
|
f104d02dc6 | ||
|
|
ac85ad1aa7 | ||
|
|
21f3c20a5f | ||
|
|
b43d877035 | ||
|
|
8f46ee052c | ||
|
|
6cda9c4a60 | ||
|
|
cfc3e491d6 | ||
|
|
23668b294f | ||
|
|
beceb678ea | ||
|
|
4257e28233 | ||
|
|
fb007c170e | ||
|
|
cabeb2c0d7 | ||
|
|
e4a2b011a4 | ||
|
|
8f6dfd1644 | ||
|
|
33fb3dc857 | ||
|
|
186edeb0f2 | ||
|
|
a034223376 | ||
|
|
9c75c75aa5 | ||
|
|
c8646a8d59 | ||
|
|
b28cb4451b | ||
|
|
a4b9a1034c | ||
|
|
12e64316eb | ||
|
|
a2389435d4 | ||
|
|
6a0455b1cd | ||
|
|
7abc9e68e4 | ||
|
|
e45dcb0450 | ||
|
|
a8ce05000e | ||
|
|
747d3e7ad1 | ||
|
|
fcf0ec0b87 | ||
|
|
976c6da07c | ||
|
|
ed96a0a8e8 | ||
|
|
d5c3bbbcc5 | ||
|
|
ec1ac9af73 | ||
|
|
2a2d11d01b | ||
|
|
14eba0fb93 | ||
|
|
418afe8f87 | ||
|
|
2b39ec3ab5 | ||
|
|
2ff895b147 | ||
|
|
16a6223c48 | ||
|
|
e928ac3746 | ||
|
|
8f5d54ebd4 | ||
|
|
123f63972e | ||
|
|
dd30d54ed9 | ||
|
|
d37bf1c824 | ||
|
|
6cd4a8bd67 | ||
|
|
1fa71e0c6a | ||
|
|
c5efeb5be9 | ||
|
|
2c8fa5da44 | ||
|
|
9cbc94eb58 | ||
|
|
4ce4482764 | ||
|
|
49a8d970fe | ||
|
|
1be6f24676 | ||
|
|
82b94f1a71 | ||
|
|
019c5789ce | ||
|
|
4336defc14 | ||
|
|
712a2100bb | ||
|
|
bc46f54604 | ||
|
|
1b713e7874 | ||
|
|
3f5dfd2219 | ||
|
|
ea707fda42 | ||
|
|
b245be476a | ||
|
|
9619530814 | ||
|
|
adf617e59f | ||
|
|
3f49855840 | ||
|
|
053d15c559 | ||
|
|
b3a479636b | ||
|
|
8539986faf | ||
|
|
cbdf28c510 | ||
|
|
670cc7769d | ||
|
|
22a9be3ba2 | ||
|
|
2836b03366 | ||
|
|
64bde2189e | ||
|
|
4d41c8e19a | ||
|
|
08a095e349 | ||
|
|
fc0f2b8356 | ||
|
|
e9c0f63478 | ||
|
|
9c01d00686 | ||
|
|
dd8d379292 | ||
|
|
1acd5ce5cb | ||
|
|
aa3ac901b6 | ||
|
|
2c6270f23e | ||
|
|
f6728b4e8d | ||
|
|
fb0590d778 | ||
|
|
f94ec78c66 | ||
|
|
3fddbb4c6f | ||
|
|
e841ac90b8 | ||
|
|
203a97d7db | ||
|
|
4d19538c77 | ||
|
|
e83356cb74 | ||
|
|
5f1beb6045 | ||
|
|
12ee77386b | ||
|
|
accbb2af9a | ||
|
|
041cee1a80 | ||
|
|
f8efdbf240 | ||
|
|
ca4155be6f | ||
|
|
b3c7b143e0 | ||
|
|
42d31efff8 | ||
|
|
56e90ef693 | ||
|
|
6345efa996 | ||
|
|
701f841cfe | ||
|
|
4b6de88e25 | ||
|
|
de9b0df49e | ||
|
|
b8418e4e19 | ||
|
|
d5cec191e2 | ||
|
|
13feefde31 | ||
|
|
86c9d8835d | ||
|
|
b8da5152e0 | ||
|
|
e73564c578 | ||
|
|
f42696ddfc | ||
|
|
7e3a85050d | ||
|
|
b022c5539e | ||
|
|
e4bdea4ee9 | ||
|
|
60325e1821 | ||
|
|
69ca6a8691 | ||
|
|
b18126ccd1 | ||
|
|
8d41f6305d | ||
|
|
ad223941e2 | ||
|
|
99b3635e76 | ||
|
|
f912458faa | ||
|
|
097cbf485b | ||
|
|
f85f77814a | ||
|
|
e7d38a07ff | ||
|
|
66a64241d9 | ||
|
|
c78a27bc71 | ||
|
|
b6cdadaaee | ||
|
|
a6b88f18ab | ||
|
|
b101208828 | ||
|
|
368a33b545 | ||
|
|
7850c49054 | ||
|
|
d97dd7971e | ||
|
|
6f61ffcf33 | ||
|
|
d6c9dc1a27 | ||
|
|
2dbd0b61b7 | ||
|
|
c7e1567e5a | ||
|
|
e661cbebc1 | ||
|
|
4e4f1911af | ||
|
|
45975462bd | ||
|
|
9851638c32 |
No files matched your search
@@ -1,9 +0,0 @@
|
||||
If this is a question or feature request, make sure to:
|
||||
|
||||
- [ ] The title starts with `Question:` or `Feature:`.
|
||||
|
||||
If this is an bug report, not a question, make sure to:
|
||||
|
||||
- [ ] I'm not running Windows (which is unsupported), or if I am, I can confirm this issue appears on another platform, or Vagrant.
|
||||
- [ ] This issue appears in the latest `dev` branch.
|
||||
- [ ] I've included my browser and Ruby version in this issue.
|
||||
@@ -1,5 +0,0 @@
|
||||
!!!!! STOP AND READ !!!!!
|
||||
|
||||
If the dropdown above says "base fork: lord/master", you are submitting your change to ALL USERS OF SLATE, not just your company. This is probably not what you want. Click "base fork" to change it to the right place.
|
||||
|
||||
If you're actually trying to submit a change to upstream Slate, please submit to our dev branch, PRs sent to the master branch are generally rejected.
|
||||
+9
-23
@@ -1,25 +1,11 @@
|
||||
*.gem
|
||||
*.rbc
|
||||
.bundle
|
||||
.config
|
||||
coverage
|
||||
InstalledFiles
|
||||
lib/bundler/man
|
||||
pkg
|
||||
rdoc
|
||||
spec/reports
|
||||
test/tmp
|
||||
test/version_tmp
|
||||
tmp
|
||||
*.DS_STORE
|
||||
build/
|
||||
.cache
|
||||
.vagrant
|
||||
.sass-cache
|
||||
|
||||
# YARD artifacts
|
||||
.yardoc
|
||||
_yardoc
|
||||
doc/
|
||||
.classpath
|
||||
.project
|
||||
.settings/
|
||||
target/
|
||||
.idea/
|
||||
*.iml
|
||||
logs/
|
||||
server/pki/
|
||||
client/src/main/python/setup.py
|
||||
grafana/timely-app/yarn-error.log
|
||||
|
||||
-10
@@ -1,10 +0,0 @@
|
||||
sudo: false
|
||||
|
||||
language: ruby
|
||||
|
||||
rvm:
|
||||
- 2.2.5
|
||||
- 2.3.1
|
||||
|
||||
cache: bundler
|
||||
script: bundle exec middleman build
|
||||
@@ -1,88 +0,0 @@
|
||||
# Changelog
|
||||
|
||||
## Version 1.4.0
|
||||
|
||||
*November 24, 2016*
|
||||
|
||||
- Upgrade Middleman and Rouge gems, should hopefully solve a number of bugs
|
||||
- Update some links in README
|
||||
- Fix broken Vagrant startup script
|
||||
- Fix some problems with deploy.sh help message
|
||||
- Fix bug with language tabs not hiding properly if no error
|
||||
- Add `!default` to SASS variables
|
||||
- Fix bug with logo margin
|
||||
- Bump tested Ruby versions in .travis.yml
|
||||
|
||||
## Version 1.3.3
|
||||
|
||||
*June 11, 2016*
|
||||
|
||||
Documentation and example changes.
|
||||
|
||||
## Version 1.3.2
|
||||
|
||||
*February 3, 2016*
|
||||
|
||||
A small bugfix for slightly incorrect background colors on code samples in some cases.
|
||||
|
||||
## Version 1.3.1
|
||||
|
||||
*January 31, 2016*
|
||||
|
||||
A small bugfix for incorrect whitespace in code blocks.
|
||||
|
||||
## Version 1.3
|
||||
|
||||
*January 27, 2016*
|
||||
|
||||
We've upgraded Middleman and a number of other dependencies, which should fix quite a few bugs.
|
||||
|
||||
Instead of `rake build` and `rake deploy`, you should now run `bundle exec middleman build --clean` to build your server, and `./deploy.sh` to deploy it to Github Pages.
|
||||
|
||||
## Version 1.2
|
||||
|
||||
*June 20, 2015*
|
||||
|
||||
**Fixes:**
|
||||
|
||||
- Remove crash on invalid languages
|
||||
- Update Tocify to scroll to the highlighted header in the Table of Contents
|
||||
- Fix variable leak and update search algorithms
|
||||
- Update Python examples to be valid Python
|
||||
- Update gems
|
||||
- More misc. bugfixes of Javascript errors
|
||||
- Add Dockerfile
|
||||
- Remove unused gems
|
||||
- Optimize images, fonts, and generated asset files
|
||||
- Add chinese font support
|
||||
- Remove RedCarpet header ID patch
|
||||
- Update language tabs to not disturb existing query strings
|
||||
|
||||
## Version 1.1
|
||||
|
||||
*July 27, 2014*
|
||||
|
||||
**Fixes:**
|
||||
|
||||
- Finally, a fix for the redcarpet upgrade bug
|
||||
|
||||
## Version 1.0
|
||||
|
||||
*July 2, 2014*
|
||||
|
||||
[View Issues](https://github.com/tripit/slate/issues?milestone=1&state=closed)
|
||||
|
||||
**Features:**
|
||||
|
||||
- Responsive designs for phones and tablets
|
||||
- Started tagging versions
|
||||
|
||||
**Fixes:**
|
||||
|
||||
- Fixed 'unrecognized expression' error
|
||||
- Fixed #undefined hash bug
|
||||
- Fixed bug where the current language tab would be unselected
|
||||
- Fixed bug where tocify wouldn't highlight the current section while searching
|
||||
- Fixed bug where ids of header tags would have special characters that caused problems
|
||||
- Updated layout so that pages with disabled search wouldn't load search.js
|
||||
- Cleaned up Javascript
|
||||
@@ -0,0 +1,6 @@
|
||||
Dave Marion ([email protected])
|
||||
Eric Newton ([email protected])
|
||||
Jim Klucar ([email protected])
|
||||
Charlie Williams ([email protected])
|
||||
Drew Farris ([email protected])
|
||||
Brian Loss ([email protected])
|
||||
@@ -1,9 +0,0 @@
|
||||
source 'https://rubygems.org'
|
||||
|
||||
# Middleman
|
||||
gem 'middleman', '~>4.1.0'
|
||||
gem 'middleman-syntax', '~> 3.0.0'
|
||||
gem 'middleman-autoprefixer', '~> 2.7.0'
|
||||
gem "middleman-sprockets", "~> 4.0.0"
|
||||
gem 'rouge', '~> 2.0.5'
|
||||
gem 'redcarpet', '~> 3.3.2'
|
||||
-122
@@ -1,122 +0,0 @@
|
||||
GEM
|
||||
remote: https://rubygems.org/
|
||||
specs:
|
||||
activesupport (4.2.7.1)
|
||||
i18n (~> 0.7)
|
||||
json (~> 1.7, >= 1.7.7)
|
||||
minitest (~> 5.1)
|
||||
thread_safe (~> 0.3, >= 0.3.4)
|
||||
tzinfo (~> 1.1)
|
||||
addressable (2.4.0)
|
||||
autoprefixer-rails (6.4.1)
|
||||
execjs
|
||||
backports (3.6.8)
|
||||
coffee-script (2.4.1)
|
||||
coffee-script-source
|
||||
execjs
|
||||
coffee-script-source (1.10.0)
|
||||
compass-import-once (1.0.5)
|
||||
sass (>= 3.2, < 3.5)
|
||||
concurrent-ruby (1.0.2)
|
||||
contracts (0.13.0)
|
||||
dotenv (2.1.1)
|
||||
erubis (2.7.0)
|
||||
execjs (2.7.0)
|
||||
fast_blank (1.0.0)
|
||||
fastimage (2.0.0)
|
||||
addressable (~> 2)
|
||||
ffi (1.9.14)
|
||||
haml (4.0.7)
|
||||
tilt
|
||||
hamster (3.0.0)
|
||||
concurrent-ruby (~> 1.0)
|
||||
hashie (3.4.4)
|
||||
i18n (0.7.0)
|
||||
json (1.8.3)
|
||||
kramdown (1.12.0)
|
||||
listen (3.0.8)
|
||||
rb-fsevent (~> 0.9, >= 0.9.4)
|
||||
rb-inotify (~> 0.9, >= 0.9.7)
|
||||
memoist (0.15.0)
|
||||
middleman (4.1.10)
|
||||
coffee-script (~> 2.2)
|
||||
compass-import-once (= 1.0.5)
|
||||
haml (>= 4.0.5)
|
||||
kramdown (~> 1.2)
|
||||
middleman-cli (= 4.1.10)
|
||||
middleman-core (= 4.1.10)
|
||||
sass (>= 3.4.0, < 4.0)
|
||||
middleman-autoprefixer (2.7.0)
|
||||
autoprefixer-rails (>= 6.3.1, < 7.0.0)
|
||||
middleman-core (>= 3.3.3)
|
||||
middleman-cli (4.1.10)
|
||||
thor (>= 0.17.0, < 2.0)
|
||||
middleman-core (4.1.10)
|
||||
activesupport (~> 4.2)
|
||||
addressable (~> 2.3)
|
||||
backports (~> 3.6)
|
||||
bundler (~> 1.1)
|
||||
contracts (~> 0.13.0)
|
||||
dotenv
|
||||
erubis
|
||||
execjs (~> 2.0)
|
||||
fast_blank
|
||||
fastimage (~> 2.0)
|
||||
hamster (~> 3.0)
|
||||
hashie (~> 3.4)
|
||||
i18n (~> 0.7.0)
|
||||
listen (~> 3.0.0)
|
||||
memoist (~> 0.14)
|
||||
padrino-helpers (~> 0.13.0)
|
||||
parallel
|
||||
rack (>= 1.4.5, < 2.0)
|
||||
sass (>= 3.4)
|
||||
servolux
|
||||
tilt (~> 1.4.1)
|
||||
uglifier (~> 3.0)
|
||||
middleman-sprockets (4.0.0)
|
||||
middleman-core (~> 4.0)
|
||||
sprockets (>= 3.0)
|
||||
middleman-syntax (3.0.0)
|
||||
middleman-core (>= 3.2)
|
||||
rouge (~> 2.0)
|
||||
minitest (5.9.0)
|
||||
padrino-helpers (0.13.3.1)
|
||||
i18n (~> 0.6, >= 0.6.7)
|
||||
padrino-support (= 0.13.3.1)
|
||||
tilt (>= 1.4.1, < 3)
|
||||
padrino-support (0.13.3.1)
|
||||
activesupport (>= 3.1)
|
||||
parallel (1.9.0)
|
||||
rack (1.6.4)
|
||||
rb-fsevent (0.9.7)
|
||||
rb-inotify (0.9.7)
|
||||
ffi (>= 0.5.0)
|
||||
redcarpet (3.3.4)
|
||||
rouge (2.0.5)
|
||||
sass (3.4.22)
|
||||
servolux (0.12.0)
|
||||
sprockets (3.7.0)
|
||||
concurrent-ruby (~> 1.0)
|
||||
rack (> 1, < 3)
|
||||
thor (0.19.1)
|
||||
thread_safe (0.3.5)
|
||||
tilt (1.4.1)
|
||||
tzinfo (1.2.2)
|
||||
thread_safe (~> 0.1)
|
||||
uglifier (3.0.2)
|
||||
execjs (>= 0.3.0, < 3)
|
||||
|
||||
PLATFORMS
|
||||
ruby
|
||||
|
||||
DEPENDENCIES
|
||||
middleman (~> 4.1.0)
|
||||
middleman-autoprefixer (~> 2.7.0)
|
||||
middleman-sprockets (~> 4.0.0)
|
||||
middleman-syntax (~> 3.0.0)
|
||||
redcarpet (~> 3.3.2)
|
||||
rouge (~> 2.0.5)
|
||||
|
||||
BUNDLED WITH
|
||||
1.12.5
|
||||
@@ -1,13 +1,203 @@
|
||||
Copyright 2008-2013 Concur Technologies, Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
not use this file except in compliance with the License. You may obtain
|
||||
a copy of the License at
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
httpProperties://www.apache.org/licenses/
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
httpProperties://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
License for the specific language governing permissions and limitations
|
||||
under the License.
|
||||
@@ -1,110 +1,8 @@
|
||||
<p align="center">
|
||||
<img src="https://raw.githubusercontent.com/lord/img/master/logo-slate.png" alt="Slate: API Documentation Generator" width="226">
|
||||
<br>
|
||||
<a href="https://travis-ci.org/lord/slate"><img src="https://travis-ci.org/lord/slate.svg?branch=master" alt="Build Status"></a>
|
||||
</p>
|
||||

|
||||
|
||||
<p align="center">Slate helps you create beautiful, intelligent, responsive API documentation.</p>
|
||||
[![Apache License][li]][ll]
|
||||
|
||||
<p align="center"><img src="https://dl.dropboxusercontent.com/u/95847291/github%20images/slate/slate_screenshot_new.png" width=700 alt="Screenshot of Example Documentation created with Slate"></p>
|
||||
Timely is a time series database application that provides secure access to time series data. Timely is written in Java and designed to work with [Apache Accumulo](https://accumuloProperties.apache.org/) and [Grafana](https://www.grafana.com). Documentation is located [here](https://nationalsecurityagency.github.io/timely/).
|
||||
|
||||
<p align="center"><em>The example above was created with Slate. Check it out at <a href="https://lord.github.io/slate">lord.github.io/slate</a>.</em></p>
|
||||
|
||||
Features
|
||||
------------
|
||||
|
||||
* **Clean, intuitive design** — With Slate, the description of your API is on the left side of your documentation, and all the code examples are on the right side. Inspired by [Stripe's](https://stripe.com/docs/api) and [Paypal's](https://developer.paypal.com/webapps/developer/docs/api/) API docs. Slate is responsive, so it looks great on tablets, phones, and even in print.
|
||||
|
||||
* **Everything on a single page** — Gone are the days when your users had to search through a million pages to find what they wanted. Slate puts the entire documentation on a single page. We haven't sacrificed linkability, though. As you scroll, your browser's hash will update to the nearest header, so linking to a particular point in the documentation is still natural and easy.
|
||||
|
||||
* **Slate is just Markdown** — When you write docs with Slate, you're just writing Markdown, which makes it simple to edit and understand. Everything is written in Markdown — even the code samples are just Markdown code blocks.
|
||||
|
||||
* **Write code samples in multiple languages** — If your API has bindings in multiple programming languages, you can easily put in tabs to switch between them. In your document, you'll distinguish different languages by specifying the language name at the top of each code block, just like with Github Flavored Markdown.
|
||||
|
||||
* **Out-of-the-box syntax highlighting** for [over 100 languages](https://github.com/jneen/rouge/wiki/List-of-supported-languages-and-lexers), no configuration required.
|
||||
|
||||
* **Automatic, smoothly scrolling table of contents** on the far left of the page. As you scroll, it displays your current position in the document. It's fast, too. We're using Slate at TripIt to build documentation for our new API, where our table of contents has over 180 entries. We've made sure that the performance remains excellent, even for larger documents.
|
||||
|
||||
* **Let your users update your documentation for you** — By default, your Slate-generated documentation is hosted in a public Github repository. Not only does this mean you get free hosting for your docs with Github Pages, but it also makes it simple for other developers to make pull requests to your docs if they find typos or other problems. Of course, if you don't want to use GitHub, you're also welcome to host your docs elsewhere.
|
||||
|
||||
Getting started with Slate is super easy! Simply fork this repository and follow the instructions below. Or, if you'd like to check out what Slate is capable of, take a look at the [sample docs](http://lord.github.io/slate).
|
||||
|
||||
Getting Started with Slate
|
||||
------------------------------
|
||||
|
||||
### Prerequisites
|
||||
|
||||
You're going to need:
|
||||
|
||||
- **Linux or OS X** — Windows may work, but is unsupported.
|
||||
- **Ruby, version 2.2.5 or newer**
|
||||
- **Bundler** — If Ruby is already installed, but the `bundle` command doesn't work, just run `gem install bundler` in a terminal.
|
||||
|
||||
### Getting Set Up
|
||||
|
||||
1. Fork this repository on Github.
|
||||
2. Clone *your forked repository* (not our original one) to your hard drive with `git clone https://github.com/YOURUSERNAME/slate.git`
|
||||
3. `cd slate`
|
||||
4. Initialize and start Slate. You can either do this locally, or with Vagrant:
|
||||
|
||||
```shell
|
||||
# either run this to run locally
|
||||
bundle install
|
||||
bundle exec middleman server
|
||||
|
||||
# OR run this to run with vagrant
|
||||
vagrant up
|
||||
```
|
||||
|
||||
You can now see the docs at http://localhost:4567. Whoa! That was fast!
|
||||
|
||||
Now that Slate is all set up on your machine, you'll probably want to learn more about [editing Slate markdown](https://github.com/lord/slate/wiki/Markdown-Syntax), or [how to publish your docs](https://github.com/lord/slate/wiki/Deploying-Slate).
|
||||
|
||||
If you'd prefer to use Docker, instructions are available [in the wiki](https://github.com/lord/slate/wiki/Docker).
|
||||
|
||||
Companies Using Slate
|
||||
---------------------------------
|
||||
|
||||
* [NASA](https://api.nasa.gov)
|
||||
* [IBM](https://docs.cloudant.com/api.html)
|
||||
* [Sony](http://developers.cimediacloud.com)
|
||||
* [Mozilla](http://mozilla.github.io/localForage/)
|
||||
* [Best Buy](https://bestbuyapis.github.io/api-documentation/)
|
||||
* [Travis-CI](https://docs.travis-ci.com/api/)
|
||||
* [Greenhouse](https://developers.greenhouse.io/harvest.html)
|
||||
* [Woocommerce](http://woothemes.github.io/woocommerce-rest-api-docs/)
|
||||
* [Appium](http://appium.io/slate/en/master)
|
||||
* [Dwolla](https://docs.dwolla.com/)
|
||||
* [Clearbit](https://clearbit.com/docs)
|
||||
* [Coinbase](https://developers.coinbase.com/api)
|
||||
* [Parrot Drones](http://developer.parrot.com/docs/bebop/)
|
||||
* [Fidor Bank](http://docs.fidor.de/)
|
||||
|
||||
You can view more in [the list on the wiki](https://github.com/lord/slate/wiki/Slate-in-the-Wild).
|
||||
|
||||
Need Help? Found a bug?
|
||||
--------------------
|
||||
|
||||
[Submit an issue](https://github.com/lord/slate/issues) to the Slate Github if you need any help. And, of course, feel free to submit pull requests with bug fixes or changes.
|
||||
|
||||
Contributors
|
||||
--------------------
|
||||
|
||||
Slate was built by [Robert Lord](https://lord.io) while interning at [TripIt](https://www.tripit.com/).
|
||||
|
||||
Thanks to the following people who have submitted major pull requests:
|
||||
|
||||
- [@chrissrogers](https://github.com/chrissrogers)
|
||||
- [@bootstraponline](https://github.com/bootstraponline)
|
||||
- [@realityking](https://github.com/realityking)
|
||||
- [@cvkef](https://github.com/cvkef)
|
||||
|
||||
Also, thanks to [Sauce Labs](http://saucelabs.com) for helping sponsor the project.
|
||||
|
||||
Special Thanks
|
||||
--------------------
|
||||
- [Middleman](https://github.com/middleman/middleman)
|
||||
- [jquery.tocify.js](https://github.com/gfranko/jquery.tocify.js)
|
||||
- [middleman-syntax](https://github.com/middleman/middleman-syntax)
|
||||
- [middleman-gh-pages](https://github.com/edgecase/middleman-gh-pages)
|
||||
- [Font Awesome](http://fortawesome.github.io/Font-Awesome/)
|
||||
[li]: https://img.shields.io/badge/license-ASL-blue.svg
|
||||
[ll]: https://www.apache.org/licenses/LICENSE-2.0
|
||||
Vendored
-39
@@ -1,39 +0,0 @@
|
||||
Vagrant.configure(2) do |config|
|
||||
config.vm.box = "ubuntu/trusty64"
|
||||
config.vm.network :forwarded_port, guest: 4567, host: 4567
|
||||
|
||||
config.vm.provision "bootstrap",
|
||||
type: "shell",
|
||||
inline: <<-SHELL
|
||||
sudo apt-get update
|
||||
sudo apt-get install -yq ruby2.0 ruby2.0-dev pkg-config build-essential nodejs git libxml2-dev libxslt-dev
|
||||
sudo apt-get autoremove -yq
|
||||
gem2.0 install --no-ri --no-rdoc bundler
|
||||
SHELL
|
||||
|
||||
# add the local user git config to the vm
|
||||
config.vm.provision "file", source: "~/.gitconfig", destination: ".gitconfig"
|
||||
|
||||
config.vm.provision "install",
|
||||
type: "shell",
|
||||
privileged: false,
|
||||
inline: <<-SHELL
|
||||
echo "=============================================="
|
||||
echo "Installing app dependencies"
|
||||
cd /vagrant
|
||||
bundle config build.nokogiri --use-system-libraries
|
||||
bundle install
|
||||
SHELL
|
||||
|
||||
config.vm.provision "run",
|
||||
type: "shell",
|
||||
privileged: false,
|
||||
run: "always",
|
||||
inline: <<-SHELL
|
||||
echo "=============================================="
|
||||
echo "Starting up middleman at http://localhost:4567"
|
||||
echo "If it does not come up, check the ~/middleman.log file for any error messages"
|
||||
cd /vagrant
|
||||
bundle exec middleman server --watcher-force-polling --watcher_latency=1 &> ~/middleman.log &
|
||||
SHELL
|
||||
end
|
||||
@@ -0,0 +1,2 @@
|
||||
/bin/
|
||||
/target
|
||||
@@ -0,0 +1,20 @@
|
||||
timelyHostname=
|
||||
timelyTcpPort=
|
||||
timelyHttpsPort=
|
||||
timelyWssPort=
|
||||
doLogin=
|
||||
#timelyUsername=
|
||||
#timelyPassword=
|
||||
keyStoreFile=
|
||||
keyStoreType=
|
||||
keyStorePass=
|
||||
trustStoreFile=
|
||||
trustStoreType=
|
||||
trustStorePass=
|
||||
hostVerificationEnabled=false
|
||||
bufferSize=10485760
|
||||
metrics=
|
||||
startTime=0
|
||||
endTime=0
|
||||
interval=1
|
||||
intervalUnits=HOURS
|
||||
@@ -0,0 +1,270 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>timely</groupId>
|
||||
<artifactId>timely</artifactId>
|
||||
<version>2.1.0</version>
|
||||
</parent>
|
||||
<artifactId>timely-analytics</artifactId>
|
||||
<!--
|
||||
|
||||
Execute "mvn clean package -Pbuild-jar"
|
||||
to build a jar file out of this project!
|
||||
|
||||
How to use the Flink Quickstart pom:
|
||||
|
||||
a) Adding new dependencies:
|
||||
You can add dependencies to the list below.
|
||||
Please check if the maven-shade-plugin below is filtering out your dependency
|
||||
and remove the exclude from there.
|
||||
|
||||
b) Build a jar for running on the cluster:
|
||||
There are two options for creating a jar from this project
|
||||
|
||||
b.1) "mvn clean package" -> this will create a fat jar which contains all
|
||||
dependencies necessary for running the jar created by this pom in a cluster.
|
||||
The "maven-shade-plugin" excludes everything that is provided on a running Flink cluster.
|
||||
|
||||
b.2) "mvn clean package -Pbuild-jar" -> This will also create a fat-jar, but with much
|
||||
nicer dependency exclusion handling. This approach is preferred and leads to
|
||||
much cleaner jar files.
|
||||
-->
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.flink</groupId>
|
||||
<artifactId>flink-clients_2.11</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.flink</groupId>
|
||||
<artifactId>flink-java</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.flink</groupId>
|
||||
<artifactId>flink-streaming-java_2.11</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>timely</groupId>
|
||||
<artifactId>timely-client</artifactId>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>*</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>timely</groupId>
|
||||
<artifactId>timely-common</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<build>
|
||||
<pluginManagement>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<configuration>
|
||||
<compilerId>jdt</compilerId>
|
||||
</configuration>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.tycho</groupId>
|
||||
<artifactId>tycho-compiler-jdt</artifactId>
|
||||
<version>1.4.0</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</pluginManagement>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
</plugin>
|
||||
<!-- We use the maven-shade plugin to create a fat jar that contains all dependencies
|
||||
except flink and it's transitive dependencies. The resulting fat-jar can be executed
|
||||
on a cluster. Change the value of Program-Class if your program entry point changes. -->
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-shade-plugin</artifactId>
|
||||
<executions>
|
||||
<!-- Run shade goal on package phase -->
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>shade</goal>
|
||||
</goals>
|
||||
<phase>package</phase>
|
||||
<configuration>
|
||||
<artifactSet>
|
||||
<excludes>
|
||||
<!-- This list contains all dependencies of flink-dist
|
||||
Everything else will be packaged into the fat-jar
|
||||
-->
|
||||
<exclude>org.apache.flink:flink-annotations</exclude>
|
||||
<exclude>org.apache.flink:flink-shaded-hadoop1_2.11</exclude>
|
||||
<exclude>org.apache.flink:flink-shaded-hadoop2</exclude>
|
||||
<exclude>org.apache.flink:flink-shaded-curator-recipes</exclude>
|
||||
<exclude>org.apache.flink:flink-core</exclude>
|
||||
<exclude>org.apache.flink:flink-java</exclude>
|
||||
<exclude>org.apache.flink:flink-scala_2.11</exclude>
|
||||
<exclude>org.apache.flink:flink-runtime_2.11</exclude>
|
||||
<exclude>org.apache.flink:flink-optimizer_2.11</exclude>
|
||||
<exclude>org.apache.flink:flink-clients_2.11</exclude>
|
||||
<exclude>org.apache.flink:flink-avro_2.11</exclude>
|
||||
<exclude>org.apache.flink:flink-examples-batch_2.11</exclude>
|
||||
<exclude>org.apache.flink:flink-examples-streaming_2.11</exclude>
|
||||
<exclude>org.apache.flink:flink-streaming-java_2.11</exclude>
|
||||
<!-- Also exclude very big transitive dependencies of Flink
|
||||
WARNING: You have to remove these excludes if your code relies on other
|
||||
versions of these dependencies.
|
||||
-->
|
||||
<exclude>org.scala-lang:scala-library</exclude>
|
||||
<exclude>org.scala-lang:scala-compiler</exclude>
|
||||
<exclude>org.scala-lang:scala-reflect</exclude>
|
||||
<exclude>com.typesafe.akka:akka-actor_*</exclude>
|
||||
<exclude>com.typesafe.akka:akka-remote_*</exclude>
|
||||
<exclude>com.typesafe.akka:akka-slf4j_*</exclude>
|
||||
<exclude>io.netty:netty-all</exclude>
|
||||
<exclude>io.netty:netty</exclude>
|
||||
<exclude>commons-fileupload:commons-fileupload</exclude>
|
||||
<exclude>org.apache.avro:avro</exclude>
|
||||
<exclude>commons-collections:commons-collections</exclude>
|
||||
<exclude>org.codehaus.jackson:jackson-core-asl</exclude>
|
||||
<exclude>org.codehaus.jackson:jackson-mapper-asl</exclude>
|
||||
<exclude>com.thoughtworks.paranamer:paranamer</exclude>
|
||||
<exclude>org.xerial.snappy:snappy-java</exclude>
|
||||
<exclude>org.apache.commons:commons-compress</exclude>
|
||||
<exclude>org.tukaani:xz</exclude>
|
||||
<exclude>com.esotericsoftware.kryo:kryo</exclude>
|
||||
<exclude>com.esotericsoftware.minlog:minlog</exclude>
|
||||
<exclude>org.objenesis:objenesis</exclude>
|
||||
<exclude>com.twitter:chill_*</exclude>
|
||||
<exclude>com.twitter:chill-java</exclude>
|
||||
<exclude>commons-lang:commons-lang</exclude>
|
||||
<exclude>junit:junit</exclude>
|
||||
<exclude>org.apache.commons:commons-lang3</exclude>
|
||||
<exclude>org.slf4j:slf4j-api</exclude>
|
||||
<exclude>org.slf4j:slf4j-log4j12</exclude>
|
||||
<exclude>log4j:log4j</exclude>
|
||||
<exclude>org.apache.commons:commons-math</exclude>
|
||||
<exclude>org.apache.sling:org.apache.sling.commons.json</exclude>
|
||||
<exclude>commons-logging:commons-logging</exclude>
|
||||
<exclude>commons-codec:commons-codec</exclude>
|
||||
<exclude>com.fasterxml.jackson.core:jackson-core</exclude>
|
||||
<exclude>com.fasterxml.jackson.core:jackson-databind</exclude>
|
||||
<exclude>com.fasterxml.jackson.core:jackson-annotations</exclude>
|
||||
<exclude>stax:stax-api</exclude>
|
||||
<exclude>com.typesafe:config</exclude>
|
||||
<exclude>org.uncommons.maths:uncommons-maths</exclude>
|
||||
<exclude>com.github.scopt:scopt_*</exclude>
|
||||
<exclude>commons-io:commons-io</exclude>
|
||||
<exclude>commons-cli:commons-cli</exclude>
|
||||
</excludes>
|
||||
</artifactSet>
|
||||
<filters>
|
||||
<filter>
|
||||
<artifact>*:*</artifact>
|
||||
<excludes>
|
||||
<exclude>**/module-info.class</exclude>
|
||||
</excludes>
|
||||
</filter>
|
||||
<filter>
|
||||
<artifact>org.apache.flink:*</artifact>
|
||||
<excludes>
|
||||
<!-- exclude shaded google but include shaded curator -->
|
||||
<exclude>org/apache/flink/shaded/com/**</exclude>
|
||||
<exclude>web-docs/**</exclude>
|
||||
</excludes>
|
||||
</filter>
|
||||
<filter>
|
||||
<!-- Do not copy the signatures in the META-INF folder.
|
||||
Otherwise, this might cause SecurityExceptions when using the JAR.
|
||||
-->
|
||||
<artifact>*:*</artifact>
|
||||
<excludes>
|
||||
<exclude>META-INF/*.SF</exclude>
|
||||
<exclude>META-INF/*.DSA</exclude>
|
||||
<exclude>META-INF/*.RSA</exclude>
|
||||
</excludes>
|
||||
</filter>
|
||||
</filters>
|
||||
<!-- If you want to use ./bin/flink run <quickstart jar> uncomment the following lines.
|
||||
This will add a Main-Class entry to the manifest file -->
|
||||
<!--
|
||||
<transformers>
|
||||
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
|
||||
<mainClass>jar.StreamingJob</mainClass>
|
||||
</transformer>
|
||||
</transformers>
|
||||
-->
|
||||
<createDependencyReducedPom>false</createDependencyReducedPom>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.codehaus.mojo</groupId>
|
||||
<artifactId>buildnumber-maven-plugin</artifactId>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
<profiles>
|
||||
<profile>
|
||||
<!-- Profile for packaging correct JAR files -->
|
||||
<id>build-jar</id>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.flink</groupId>
|
||||
<artifactId>flink-clients_2.11</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.flink</groupId>
|
||||
<artifactId>flink-java</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.flink</groupId>
|
||||
<artifactId>flink-streaming-java_2.11</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<build>
|
||||
<plugins>
|
||||
<!-- disable the exclusion rules -->
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-shade-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>shade</goal>
|
||||
</goals>
|
||||
<phase>package</phase>
|
||||
<configuration>
|
||||
<artifactSet>
|
||||
<excludes combine.self="override" />
|
||||
</artifactSet>
|
||||
<filters>
|
||||
<filter>
|
||||
<artifact>*:*</artifact>
|
||||
<excludes>
|
||||
<exclude>**/module-info.class</exclude>
|
||||
</excludes>
|
||||
</filter>
|
||||
</filters>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
</profiles>
|
||||
</project>
|
||||
@@ -0,0 +1,169 @@
|
||||
package timely.analytics.flink;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.flink.streaming.util.serialization.SerializationSchema;
|
||||
|
||||
import timely.model.Tag;
|
||||
|
||||
public class MetricHistogram implements Serializable, SerializationSchema<MetricHistogram> {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
private static final String SPACE = " ";
|
||||
private static final String AVG = " sample=avg\n";
|
||||
private static final String COUNT = " sample=count\n";
|
||||
private static final String MAX = " sample=max\n";
|
||||
private static final String MIN = " sample=min\n";
|
||||
private static final String SUM = " sample=sum\n";
|
||||
private static final String p50 = " sample=50p\n";
|
||||
private static final String p75 = " sample=75p\n";
|
||||
private static final String p90 = " sample=90p\n";
|
||||
private static final String p99 = " sample=99p\n";
|
||||
|
||||
private String metric = null;
|
||||
private List<Tag> tags = null;
|
||||
private long timestamp = 0L;
|
||||
private boolean initialized = false;
|
||||
private List<Double> values = new ArrayList<>();
|
||||
|
||||
public void initialize(String metric, List<Tag> tags) {
|
||||
this.metric = metric;
|
||||
this.tags = tags;
|
||||
initialized = true;
|
||||
}
|
||||
|
||||
public boolean isInitialized() {
|
||||
return initialized;
|
||||
}
|
||||
|
||||
public String getMetric() {
|
||||
return metric;
|
||||
}
|
||||
|
||||
public List<Tag> getTags() {
|
||||
return tags;
|
||||
}
|
||||
|
||||
public long getTimestamp() {
|
||||
return timestamp;
|
||||
}
|
||||
|
||||
public void update(Double d, long timestamp) {
|
||||
values.add(d);
|
||||
this.timestamp = timestamp;
|
||||
}
|
||||
|
||||
public void done() {
|
||||
Collections.sort(values);
|
||||
}
|
||||
|
||||
public Double avg() {
|
||||
if (values.size() == 0) {
|
||||
return 0.0D;
|
||||
}
|
||||
return sum() / count();
|
||||
}
|
||||
|
||||
public Double min() {
|
||||
if (values.size() == 0) {
|
||||
return 0.0D;
|
||||
}
|
||||
return values.get(0);
|
||||
}
|
||||
|
||||
public Double max() {
|
||||
if (values.size() == 0) {
|
||||
return 0.0D;
|
||||
}
|
||||
return values.get(values.size() - 1);
|
||||
}
|
||||
|
||||
public Double getPercentile(int p) {
|
||||
if (values.size() == 0) {
|
||||
return 0.0D;
|
||||
}
|
||||
int position = Math.round((p * 1.0f / 100) * values.size());
|
||||
position = (0 == position) ? 0 : position - 1;
|
||||
return values.get(position);
|
||||
}
|
||||
|
||||
public Double count() {
|
||||
return values.size() * 1.0D;
|
||||
}
|
||||
|
||||
public Double sum() {
|
||||
double sum = 0.0D;
|
||||
for (Double d : values) {
|
||||
sum += d;
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
done();
|
||||
StringBuilder buf = new StringBuilder();
|
||||
buf.append("metric: ").append(metric);
|
||||
buf.append(" tags: ").append(tags);
|
||||
buf.append(" timestamp: ").append(timestamp);
|
||||
buf.append(" count: ").append(count());
|
||||
buf.append(" min: ").append(min());
|
||||
buf.append(" max: ").append(max());
|
||||
buf.append(" sum: ").append(sum());
|
||||
buf.append(" 50p: ").append(getPercentile(50));
|
||||
buf.append(" 75p: ").append(getPercentile(75));
|
||||
buf.append(" 99p: ").append(getPercentile(99));
|
||||
return buf.toString();
|
||||
}
|
||||
|
||||
public byte[] serialize(MetricHistogram histo) {
|
||||
StringBuilder buf = new StringBuilder();
|
||||
|
||||
String tags = histo.getTags().stream().map(t -> t.join()).collect(Collectors.joining(" "));
|
||||
|
||||
// @formatter:off
|
||||
buf.append("put ").append(histo.getMetric() + "_summarized")
|
||||
.append(SPACE).append(histo.getTimestamp())
|
||||
.append(SPACE).append(histo.min())
|
||||
.append(SPACE).append(tags).append(MIN);
|
||||
buf.append("put ").append(histo.getMetric() + "_summarized")
|
||||
.append(SPACE).append(histo.getTimestamp())
|
||||
.append(SPACE).append(histo.max())
|
||||
.append(SPACE).append(tags).append(MAX);
|
||||
buf.append("put ").append(histo.getMetric() + "_summarized")
|
||||
.append(SPACE).append(histo.getTimestamp())
|
||||
.append(SPACE).append(histo.count())
|
||||
.append(SPACE).append(tags).append(COUNT);
|
||||
buf.append("put ").append(histo.getMetric() + "_summarized")
|
||||
.append(SPACE).append(histo.getTimestamp())
|
||||
.append(SPACE).append(histo.sum())
|
||||
.append(SPACE).append(tags).append(SUM);
|
||||
buf.append("put ").append(histo.getMetric() + "_summarized")
|
||||
.append(SPACE).append(histo.getTimestamp())
|
||||
.append(SPACE).append(histo.avg())
|
||||
.append(SPACE).append(tags).append(AVG);
|
||||
buf.append("put ").append(histo.getMetric() + "_summarized")
|
||||
.append(SPACE).append(histo.getTimestamp())
|
||||
.append(SPACE).append(histo.getPercentile(50))
|
||||
.append(SPACE).append(tags).append(p50);
|
||||
buf.append("put ").append(histo.getMetric() + "_summarized")
|
||||
.append(SPACE).append(histo.getTimestamp())
|
||||
.append(SPACE).append(histo.getPercentile(75))
|
||||
.append(SPACE).append(tags).append(p75);
|
||||
buf.append("put ").append(histo.getMetric() + "_summarized")
|
||||
.append(SPACE).append(histo.getTimestamp())
|
||||
.append(SPACE).append(histo.getPercentile(90))
|
||||
.append(SPACE).append(tags).append(p90);
|
||||
buf.append("put ").append(histo.getMetric() + "_summarized")
|
||||
.append(SPACE).append(histo.getTimestamp())
|
||||
.append(SPACE).append(histo.getPercentile(99))
|
||||
.append(SPACE).append(tags).append(p99);
|
||||
// @formatter:off
|
||||
return buf.toString().getBytes();
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package timely.analytics.flink;
|
||||
|
||||
import org.apache.flink.api.java.functions.KeySelector;
|
||||
|
||||
import timely.api.response.MetricResponse;
|
||||
|
||||
/**
|
||||
* Uniquely defines a time series in a Flink stream. A unique time series is the metric name and all of its tags, to include visibility if it exists.
|
||||
*/
|
||||
public class MetricKeySelector implements KeySelector<MetricResponse,String> {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
@Override
|
||||
public String getKey(MetricResponse m) throws Exception {
|
||||
return m.getMetric() + m.getTags().toString();
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package timely.analytics.flink;
|
||||
|
||||
import java.util.concurrent.ConcurrentSkipListMap;
|
||||
|
||||
import org.apache.commons.lang3.mutable.MutableLong;
|
||||
import org.apache.flink.api.common.accumulators.Accumulator;
|
||||
|
||||
/**
|
||||
* Custom accumulator that retains a sorted map of strings and their counts
|
||||
*
|
||||
*/
|
||||
public class SortedStringAccumulator implements Accumulator<String,ConcurrentSkipListMap<String,MutableLong>> {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private ConcurrentSkipListMap<String,MutableLong> values = new ConcurrentSkipListMap<>();
|
||||
|
||||
@Override
|
||||
public void add(String value) {
|
||||
if (values.containsKey(value)) {
|
||||
values.get(value).increment();
|
||||
} else {
|
||||
values.put(value, new MutableLong(1));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public ConcurrentSkipListMap<String,MutableLong> getLocalValue() {
|
||||
return values;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void resetLocal() {
|
||||
values.clear();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void merge(Accumulator<String,ConcurrentSkipListMap<String,MutableLong>> other) {
|
||||
other.getLocalValue().forEach((k, v) -> {
|
||||
if (values.containsKey(k)) {
|
||||
values.get(k).add(v.longValue());
|
||||
} else {
|
||||
values.put(k, v);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public Accumulator<String,ConcurrentSkipListMap<String,MutableLong>> clone() {
|
||||
SortedStringAccumulator a = new SortedStringAccumulator();
|
||||
a.getLocalValue().putAll(this.values);
|
||||
return a;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
package timely.analytics.flink;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Serializable;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.Date;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.RejectedExecutionException;
|
||||
import java.util.concurrent.ThreadFactory;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import javax.websocket.CloseReason;
|
||||
import javax.websocket.EndpointConfig;
|
||||
import javax.websocket.MessageHandler;
|
||||
import javax.websocket.Session;
|
||||
|
||||
import org.apache.flink.api.common.accumulators.LongCounter;
|
||||
import org.apache.flink.api.common.functions.StoppableFunction;
|
||||
import org.apache.flink.configuration.Configuration;
|
||||
import org.apache.flink.streaming.api.functions.source.RichSourceFunction;
|
||||
import org.apache.flink.streaming.api.watermark.Watermark;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import timely.api.response.MetricResponse;
|
||||
import timely.api.response.MetricResponses;
|
||||
import timely.client.websocket.ClientHandler;
|
||||
import timely.client.websocket.subscription.WebSocketSubscriptionClient;
|
||||
import timely.serialize.JsonSerializer;
|
||||
|
||||
public class SubscriptionSource extends RichSourceFunction<MetricResponse> implements Serializable, StoppableFunction {
|
||||
|
||||
private static class SubscriptionThreadFactory implements ThreadFactory {
|
||||
|
||||
private static final String NAME = "Subscription Source";
|
||||
private AtomicInteger threadNum = new AtomicInteger(1);
|
||||
|
||||
@Override
|
||||
public Thread newThread(Runnable r) {
|
||||
return new Thread(r, NAME + " " + threadNum.getAndIncrement());
|
||||
}
|
||||
}
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(SubscriptionSource.class);
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private transient WebSocketSubscriptionClient client = null;
|
||||
private long start = 0L;
|
||||
private long end = 0L;
|
||||
private final String[] metrics;
|
||||
private final SummarizationJobParameters jp;
|
||||
private final long window;
|
||||
private final AtomicInteger subscriptionsRemaining;
|
||||
|
||||
public SubscriptionSource(SummarizationJobParameters jp) {
|
||||
this.jp = jp;
|
||||
start = jp.getStartTime();
|
||||
end = jp.getEndTime();
|
||||
metrics = jp.getMetrics();
|
||||
window = jp.getSummarizationInterval().toMilliseconds();
|
||||
subscriptionsRemaining = new AtomicInteger(metrics.length);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void open(Configuration parameters) throws Exception {
|
||||
log.info("Opening summarization job.");
|
||||
super.open(parameters);
|
||||
client = new WebSocketSubscriptionClient(jp.getTimelyHostname(), jp.getTimelyHttpsPort(), jp.getTimelyWssPort(), jp.isClientAuth(), jp.isDoLogin(),
|
||||
jp.getKeyStoreFile(), jp.getKeyStoreType(), jp.getKeyStorePass(), jp.getTrustStoreFile(), jp.getTrustStoreType(),
|
||||
jp.getTrustStorePass(), jp.isHostVerificationEnabled(), jp.getBufferSize());
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void close() throws Exception {
|
||||
super.close();
|
||||
if (null != client) {
|
||||
client.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(SourceContext<MetricResponse> ctx) throws Exception {
|
||||
log.info("Running summarization job");
|
||||
final SortedStringAccumulator dateTimeAccumulator = new SortedStringAccumulator();
|
||||
this.getRuntimeContext().addAccumulator("source hourly count", dateTimeAccumulator);
|
||||
final LongCounter sourceInputs = new LongCounter();
|
||||
this.getRuntimeContext().addAccumulator("source input counter", sourceInputs);
|
||||
final LongCounter inputProcessedInMainThread = new LongCounter();
|
||||
this.getRuntimeContext().addAccumulator("input processed in main thread", inputProcessedInMainThread);
|
||||
|
||||
// Use only one thread to ensure that time is still ordered
|
||||
final ExecutorService svc = Executors.newFixedThreadPool(1, new SubscriptionThreadFactory());
|
||||
|
||||
ClientHandler handler = new ClientHandler() {
|
||||
|
||||
@Override
|
||||
public void onOpen(Session session, EndpointConfig config) {
|
||||
session.addMessageHandler(String.class, new MessageHandler.Whole<String>() {
|
||||
|
||||
private final SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd-HH");
|
||||
private long lastWatermarkTime = 0L;
|
||||
|
||||
@Override
|
||||
public void onMessage(final String message) {
|
||||
log.info("Message received on Websocket session {}, length: {}", session.getId(), message.length());
|
||||
try {
|
||||
// Deserialize in this thread
|
||||
final MetricResponses responses = JsonSerializer.getObjectMapper().readValue(message, MetricResponses.class);
|
||||
// Process in the other
|
||||
final Runnable r = () -> {
|
||||
responses.getResponses().forEach(response -> {
|
||||
if (response.isComplete()) {
|
||||
log.info("Received last message.");
|
||||
ctx.emitWatermark(new Watermark(Long.MAX_VALUE));
|
||||
subscriptionsRemaining.getAndDecrement();
|
||||
return;
|
||||
}
|
||||
log.trace("Received metric: {}", response);
|
||||
long time = response.getTimestamp();
|
||||
ctx.collectWithTimestamp(response, time);
|
||||
dateTimeAccumulator.add(formatter.format(new Date(response.getTimestamp())));
|
||||
sourceInputs.add(1);
|
||||
// Emit a watermark every second of event
|
||||
// time
|
||||
if (lastWatermarkTime == 0) {
|
||||
lastWatermarkTime = time;
|
||||
} else if ((time - lastWatermarkTime) > window) {
|
||||
lastWatermarkTime = time;
|
||||
ctx.emitWatermark(new Watermark(time - 1));
|
||||
}
|
||||
});
|
||||
};
|
||||
try {
|
||||
svc.execute(r);
|
||||
} catch (RejectedExecutionException e) {
|
||||
// Run it in this thread
|
||||
r.run();
|
||||
inputProcessedInMainThread.add(1);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
log.error("Error deserializing metric response, closing", e);
|
||||
try {
|
||||
close();
|
||||
} catch (Exception e1) {
|
||||
log.error("Error closing client", e1);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onClose(Session session, CloseReason reason) {
|
||||
super.onClose(session, reason);
|
||||
try {
|
||||
close();
|
||||
} catch (Exception e1) {
|
||||
log.error("Error closing client", e1);
|
||||
}
|
||||
// Signal done sending data
|
||||
ctx.emitWatermark(new Watermark(Long.MAX_VALUE));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onError(Session session, Throwable error) {
|
||||
super.onError(session, error);
|
||||
try {
|
||||
close();
|
||||
} catch (Exception e1) {
|
||||
log.error("Error closing client", e1);
|
||||
}
|
||||
}
|
||||
|
||||
};
|
||||
try {
|
||||
client.open(handler);
|
||||
for (String m : metrics) {
|
||||
log.info("Adding subscription for {}", m);
|
||||
client.addSubscription(m, null, start, end, 5000);
|
||||
}
|
||||
while (!client.isClosed() && subscriptionsRemaining.get() > 0) {
|
||||
try {
|
||||
Thread.sleep(5000);
|
||||
} catch (InterruptedException e) {
|
||||
log.error("Sleep interrupted.");
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
svc.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void cancel() {
|
||||
log.info("Cancelling subscription source.");
|
||||
try {
|
||||
close();
|
||||
} catch (Exception e) {
|
||||
log.error("Error closing web socket client");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void stop() {
|
||||
log.info("Stopping subscription source.");
|
||||
try {
|
||||
close();
|
||||
} catch (Exception e) {
|
||||
log.error("Error closing web socket client");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package timely.analytics.flink;
|
||||
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.Date;
|
||||
|
||||
import org.apache.flink.api.common.accumulators.LongCounter;
|
||||
import org.apache.flink.api.common.functions.FoldFunction;
|
||||
import org.apache.flink.api.java.utils.ParameterTool;
|
||||
import org.apache.flink.configuration.Configuration;
|
||||
import org.apache.flink.streaming.api.TimeCharacteristic;
|
||||
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
|
||||
import org.apache.flink.streaming.api.functions.sink.SocketClientSink;
|
||||
import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import timely.api.response.MetricResponse;
|
||||
|
||||
public class SummarizationJob {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(SummarizationJob.class);
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
|
||||
if (args.length != 1) {
|
||||
System.err.println("Usage: timely.analytics.flink.SummarizationJob <path_to_properties_file>");
|
||||
System.exit(1);
|
||||
}
|
||||
|
||||
log.info("Loading properties from: {}", args[0]);
|
||||
ParameterTool params = ParameterTool.fromPropertiesFile(args[0]);
|
||||
|
||||
final SummarizationJobParameters jp = new SummarizationJobParameters(params);
|
||||
log.info("Properties: {}", jp);
|
||||
|
||||
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
|
||||
// @formatter:off
|
||||
env.getConfig().setGlobalJobParameters(jp);
|
||||
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
|
||||
env.addSource(new SubscriptionSource(jp), "Timely Metrics Subscription")
|
||||
// partition by unique metric
|
||||
.keyBy(new MetricKeySelector())
|
||||
.window(TumblingEventTimeWindows.of(jp.getSummarizationInterval()))
|
||||
.fold(new MetricHistogram(), new FoldFunction<MetricResponse, MetricHistogram>() {
|
||||
|
||||
private final Logger log = LoggerFactory
|
||||
.getLogger("timely.analytics.SummarizationJob.FoldFunction@");
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
@Override
|
||||
public MetricHistogram fold(MetricHistogram histo, MetricResponse metric) throws Exception {
|
||||
if (!histo.isInitialized()) {
|
||||
histo.initialize(metric.getMetric(), metric.getTags());
|
||||
} else {
|
||||
if (!histo.getMetric().equals(metric.getMetric())) {
|
||||
throw new RuntimeException("Metric name does not match - partitioning problem");
|
||||
}
|
||||
if (!histo.getTags().equals(metric.getTags())) {
|
||||
throw new RuntimeException("Tags don't not match - partitioning problem");
|
||||
}
|
||||
}
|
||||
log.trace("Updating histogram for {} with value {} at {} for metric {}", histo.getMetric()
|
||||
+ histo.getTags(), metric.getValue(), metric.getTimestamp(), metric.getMetric()
|
||||
+ metric.getTags());
|
||||
histo.update(metric.getValue(), metric.getTimestamp());
|
||||
return histo;
|
||||
}
|
||||
})
|
||||
.name("Metric Windows")
|
||||
.addSink(
|
||||
new SocketClientSink<MetricHistogram>(jp.getTimelyHostname(), jp.getTimelyTcpPort(),
|
||||
new MetricHistogram(), 2, true) {
|
||||
|
||||
private final Logger log = LoggerFactory
|
||||
.getLogger("timely.analytics.SummarizationJob.SocketClientSink");
|
||||
private static final long serialVersionUID = 1L;
|
||||
private final LongCounter sinkCounter = new LongCounter();
|
||||
private final SortedStringAccumulator dateTimeAccumulator = new SortedStringAccumulator();
|
||||
private final SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd-HH");
|
||||
|
||||
@Override
|
||||
public void open(Configuration parameters) throws Exception {
|
||||
super.open(parameters);
|
||||
this.getRuntimeContext().addAccumulator("sink counter", this.sinkCounter);
|
||||
this.getRuntimeContext().addAccumulator("sink hourly count", dateTimeAccumulator);
|
||||
log.info("Sink opened");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void invoke(MetricHistogram histo) throws Exception {
|
||||
histo.done();
|
||||
super.invoke(histo);
|
||||
this.sinkCounter.add(1);
|
||||
this.dateTimeAccumulator.add(formatter.format(new Date(histo.getTimestamp())));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws Exception {
|
||||
super.close();
|
||||
log.info("Sink closed");
|
||||
}
|
||||
|
||||
}).name("Timely Metric Sink");
|
||||
// @formatter:on
|
||||
env.execute("Metric Summarization");
|
||||
// Looks like there is no way to stop Flink job if underlying web socket
|
||||
// is closed.
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
package timely.analytics.flink;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.apache.flink.api.common.ExecutionConfig.GlobalJobParameters;
|
||||
import org.apache.flink.api.java.utils.ParameterTool;
|
||||
import org.apache.flink.configuration.Configuration;
|
||||
import org.apache.flink.streaming.api.windowing.time.Time;
|
||||
|
||||
public class SummarizationJobParameters extends GlobalJobParameters implements Serializable {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private final String timelyHostname;
|
||||
private final int timelyTcpPort;
|
||||
private final int timelyHttpsPort;
|
||||
private final int timelyWssPort;
|
||||
private final boolean clientAuth;
|
||||
private final boolean doLogin;
|
||||
private final String timelyUsername;
|
||||
private final String timelyPassword;
|
||||
private final String keyStoreFile;
|
||||
private final String keyStoreType;
|
||||
private final String keyStorePass;
|
||||
private final String trustStoreFile;
|
||||
private final String trustStoreType;
|
||||
private final String trustStorePass;
|
||||
private final boolean hostVerificationEnabled;
|
||||
private final int bufferSize;
|
||||
private final String[] metrics;
|
||||
private final long startTime;
|
||||
private final long endTime;
|
||||
private final String interval;
|
||||
private final String intervalUnits;
|
||||
|
||||
public SummarizationJobParameters(ParameterTool params) {
|
||||
timelyHostname = params.getRequired("timelyHostname");
|
||||
timelyTcpPort = params.getInt("timelyTcpPort", 4241);
|
||||
timelyHttpsPort = params.getInt("timelyHttpsPort", 4242);
|
||||
timelyWssPort = params.getInt("timelyWssPort", 4243);
|
||||
clientAuth = params.getBoolean("clientAuth", false);
|
||||
doLogin = params.getBoolean("doLogin", false);
|
||||
timelyUsername = params.get("timelyUsername", null);
|
||||
timelyPassword = params.get("timelyPassword", null);
|
||||
keyStoreFile = params.getRequired("keyStoreFile");
|
||||
keyStoreType = params.get("keyStoreType", "JKS");
|
||||
keyStorePass = params.getRequired("keyStorePass");
|
||||
trustStoreFile = params.getRequired("trustStoreFile");
|
||||
trustStoreType = params.get("trustStoreType", "JKS");
|
||||
trustStorePass = params.getRequired("trustStorePass");
|
||||
hostVerificationEnabled = params.getBoolean("hostVerificationEnabled", true);
|
||||
bufferSize = params.getInt("bufferSize", 10485760);
|
||||
String metricNames = params.getRequired("metrics");
|
||||
if (null != metricNames) {
|
||||
metrics = metricNames.split(",");
|
||||
} else {
|
||||
metrics = null;
|
||||
}
|
||||
startTime = params.getLong("startTime", 0L);
|
||||
endTime = params.getLong("endTime", 0L);
|
||||
interval = params.getRequired("interval");
|
||||
intervalUnits = params.getRequired("intervalUnits");
|
||||
}
|
||||
|
||||
public SummarizationJobParameters(Configuration conf) {
|
||||
System.out.println("Creating job parameters from configuration: " + conf);
|
||||
timelyHostname = conf.getString("timelyHostname", null);
|
||||
timelyTcpPort = conf.getInteger("timelyTcpPort", 4241);
|
||||
timelyHttpsPort = conf.getInteger("timelyHttpsPort", 4242);
|
||||
timelyWssPort = conf.getInteger("timelyWssPort", 4243);
|
||||
clientAuth = conf.getBoolean("clientAuth", false);
|
||||
doLogin = conf.getBoolean("doLogin", false);
|
||||
timelyUsername = conf.getString("timelyUsername", null);
|
||||
timelyPassword = conf.getString("timelyPassword", null);
|
||||
keyStoreFile = conf.getString("keyStoreFile", null);
|
||||
keyStoreType = conf.getString("keyStoreType", "JKS");
|
||||
keyStorePass = conf.getString("keyStorePass", null);
|
||||
trustStoreFile = conf.getString("trustStoreFile", null);
|
||||
trustStoreType = conf.getString("trustStoreType", "JKS");
|
||||
trustStorePass = conf.getString("trustStorePass", null);
|
||||
hostVerificationEnabled = conf.getBoolean("hostVerificationEnabled", true);
|
||||
bufferSize = conf.getInteger("bufferSize", 10485760);
|
||||
String metricNames = conf.getString("metrics", null);
|
||||
if (null != metricNames) {
|
||||
metrics = metricNames.split(",");
|
||||
} else {
|
||||
metrics = null;
|
||||
}
|
||||
startTime = conf.getLong("startTime", 0L);
|
||||
endTime = conf.getLong("endTime", 0L);
|
||||
interval = conf.getString("interval", null);
|
||||
intervalUnits = conf.getString("intervalUnits", null);
|
||||
}
|
||||
|
||||
public Time getSummarizationInterval() {
|
||||
return Time.of(Long.parseLong(interval), TimeUnit.valueOf(intervalUnits));
|
||||
}
|
||||
|
||||
public String getTimelyHostname() {
|
||||
return timelyHostname;
|
||||
}
|
||||
|
||||
public int getTimelyTcpPort() {
|
||||
return timelyTcpPort;
|
||||
}
|
||||
|
||||
public int getTimelyHttpsPort() {
|
||||
return timelyHttpsPort;
|
||||
}
|
||||
|
||||
public int getTimelyWssPort() {
|
||||
return timelyWssPort;
|
||||
}
|
||||
|
||||
public boolean isClientAuth() {
|
||||
return clientAuth;
|
||||
}
|
||||
|
||||
public boolean isDoLogin() {
|
||||
return doLogin;
|
||||
}
|
||||
|
||||
public String getTimelyUsername() {
|
||||
return timelyUsername;
|
||||
}
|
||||
|
||||
public String getTimelyPassword() {
|
||||
return timelyPassword;
|
||||
}
|
||||
|
||||
public String getKeyStoreFile() {
|
||||
return keyStoreFile;
|
||||
}
|
||||
|
||||
public String getKeyStoreType() {
|
||||
return keyStoreType;
|
||||
}
|
||||
|
||||
public String getKeyStorePass() {
|
||||
return keyStorePass;
|
||||
}
|
||||
|
||||
public String getTrustStoreFile() {
|
||||
return trustStoreFile;
|
||||
}
|
||||
|
||||
public String getTrustStoreType() {
|
||||
return trustStoreType;
|
||||
}
|
||||
|
||||
public String getTrustStorePass() {
|
||||
return trustStorePass;
|
||||
}
|
||||
|
||||
public boolean isHostVerificationEnabled() {
|
||||
return hostVerificationEnabled;
|
||||
}
|
||||
|
||||
public int getBufferSize() {
|
||||
return bufferSize;
|
||||
}
|
||||
|
||||
public String[] getMetrics() {
|
||||
return metrics;
|
||||
}
|
||||
|
||||
public long getStartTime() {
|
||||
return startTime;
|
||||
}
|
||||
|
||||
public long getEndTime() {
|
||||
return endTime;
|
||||
}
|
||||
|
||||
public String getInterval() {
|
||||
return interval;
|
||||
}
|
||||
|
||||
public String getIntervalUnits() {
|
||||
return intervalUnits;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String,String> toMap() {
|
||||
Map<String,String> m = new HashMap<>();
|
||||
m.put("timelyHostname", this.timelyHostname);
|
||||
m.put("timelyHttpsPort", Integer.toString(this.timelyHttpsPort));
|
||||
m.put("timelyWssPort", Integer.toString(this.timelyWssPort));
|
||||
m.put("doLogin", Boolean.toString(this.doLogin));
|
||||
m.put("timelyUsername", this.timelyUsername);
|
||||
m.put("timelyPassword", this.timelyPassword);
|
||||
m.put("keyStoreFile", this.keyStoreFile);
|
||||
m.put("keyStoreType", this.keyStoreType);
|
||||
m.put("keyStorePass", this.keyStorePass);
|
||||
m.put("trustStoreFile", this.trustStoreFile);
|
||||
m.put("trustStoreType", this.trustStoreType);
|
||||
m.put("trustStorePass", this.trustStorePass);
|
||||
m.put("hostVerificationEnabled", Boolean.toString(this.hostVerificationEnabled));
|
||||
m.put("bufferSize", Integer.toString(this.bufferSize));
|
||||
m.put("metrics", String.join(",", this.metrics));
|
||||
m.put("startTime", Long.toString(this.startTime));
|
||||
m.put("endTime", Long.toString(this.endTime));
|
||||
m.put("interval", this.interval);
|
||||
m.put("intervalUnits", this.intervalUnits);
|
||||
return m;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return toMap().toString();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<FindBugsFilter>
|
||||
<Match>
|
||||
<Class name="timely.analytics.flink.MetricHistogram" />
|
||||
<Method name="serialize" />
|
||||
<Bug pattern="DM_DEFAULT_ENCODING" />
|
||||
</Match>
|
||||
<Match>
|
||||
<Class name="timely.analytics.flink.SortedStringAccumulator" />
|
||||
<Method name="clone" />
|
||||
<Bug pattern="CN_IDIOM_NO_SUPER_CALL" />
|
||||
</Match>
|
||||
<Match>
|
||||
<Class name="timely.analytics.flink.SummarizationJobParameters" />
|
||||
<Method name="getMetrics" />
|
||||
<Bug pattern="EI_EXPOSE_REP" />
|
||||
</Match>
|
||||
<Match>
|
||||
<!-- these are probably avoidable, but needs further triage to fix -->
|
||||
<Bug pattern="EI_EXPOSE_REP" />
|
||||
</Match>
|
||||
<Match>
|
||||
<!-- these are probably avoidable, but needs further triage to fix -->
|
||||
<Bug pattern="EI_EXPOSE_REP2" />
|
||||
</Match>
|
||||
</FindBugsFilter>
|
||||
@@ -0,0 +1,106 @@
|
||||
package timely.analytics.flink;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
import timely.model.Metric;
|
||||
import timely.model.Tag;
|
||||
import timely.model.parse.MetricParser;
|
||||
|
||||
public class MetricHistogramTest {
|
||||
|
||||
private MetricHistogram m = null;
|
||||
private long timestamp = System.currentTimeMillis();
|
||||
|
||||
@Before
|
||||
public void createMetric() throws Exception {
|
||||
m = new MetricHistogram();
|
||||
for (double d = 1.0D; d <= 100.0; d += 1.0) {
|
||||
timestamp += 1000;
|
||||
m.update(d, timestamp);
|
||||
}
|
||||
m.toString();
|
||||
Assert.assertEquals(timestamp, m.getTimestamp());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMin() throws Exception {
|
||||
Assert.assertEquals(1.0D, m.min(), 0.0D);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMax() throws Exception {
|
||||
Assert.assertEquals(100.0D, m.max(), 0.0D);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAvg() throws Exception {
|
||||
int sum = 0;
|
||||
for (int i = 1; i <= 100; i++) {
|
||||
sum += i;
|
||||
}
|
||||
Assert.assertEquals((sum / 100.0D), m.avg(), 0.0D);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCount() throws Exception {
|
||||
Assert.assertEquals(100.0D, m.count(), 0.0D);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test50thPercentile() throws Exception {
|
||||
Assert.assertEquals(50.0D, m.getPercentile(50), 0.0D);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test75thPercentile() throws Exception {
|
||||
Assert.assertEquals(75.0D, m.getPercentile(75), 0.0D);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test90thPercentile() throws Exception {
|
||||
Assert.assertEquals(90.0D, m.getPercentile(90), 0.0D);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test99thPercentile() throws Exception {
|
||||
Assert.assertEquals(99.0D, m.getPercentile(99), 0.0D);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSerialization() throws Exception {
|
||||
MetricParser metricParser = new MetricParser();
|
||||
Tag t1 = new Tag("tag1=value1");
|
||||
Tag t2 = new Tag("tag2=value2");
|
||||
Tag avg = new Tag("sample=avg");
|
||||
Tag min = new Tag("sample=min");
|
||||
Tag max = new Tag("sample=max");
|
||||
Tag sum = new Tag("sample=sum");
|
||||
Tag count = new Tag("sample=count");
|
||||
Tag p50 = new Tag("sample=50p");
|
||||
Tag p75 = new Tag("sample=75p");
|
||||
Tag p90 = new Tag("sample=90p");
|
||||
Tag p99 = new Tag("sample=99p");
|
||||
|
||||
List<Tag> tags = new ArrayList<>();
|
||||
tags.add(t1);
|
||||
tags.add(t2);
|
||||
m.initialize("sys.cpu.user", tags);
|
||||
|
||||
byte[] bytes = m.serialize(m);
|
||||
String puts = new String(bytes);
|
||||
for (String put : puts.split("\n")) {
|
||||
Metric metric = metricParser.parse(put);
|
||||
Assert.assertEquals("sys.cpu.user_summarized", metric.getName());
|
||||
metric.getTags().forEach(t -> {
|
||||
Assert.assertTrue(t.equals(t1) || t.equals(t2) || t.equals(avg) || t.equals(min) || t.equals(max) || t.equals(sum) || t.equals(count)
|
||||
|| t.equals(p50) || t.equals(p75) || t.equals(p90) || t.equals(p99));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Executable
+18
@@ -0,0 +1,18 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [[ `uname` == "Darwin" ]]; then
|
||||
THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
|
||||
else
|
||||
THIS_SCRIPT=`readlink -f $0`
|
||||
fi
|
||||
|
||||
THIS_DIR="${THIS_SCRIPT%/*}"
|
||||
BASE_DIR=$(cd $THIS_DIR/.. && pwd)
|
||||
BIN_DIR=${BASE_DIR}/bin
|
||||
|
||||
. ${BIN_DIR}/timely-balancer-env.sh
|
||||
|
||||
for i in $(seq 1 $NUM_INSTANCES); do
|
||||
echo ${BIN_DIR}/timely-balancer.sh $i
|
||||
${BIN_DIR}/timely-balancer.sh $i
|
||||
done
|
||||
Executable
+21
@@ -0,0 +1,21 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [[ `uname` == "Darwin" ]]; then
|
||||
THIS_SCRIPT=`python -c 'import os,sys; print os.path.realpath(sys.argv[1])' $0`
|
||||
else
|
||||
THIS_SCRIPT=`readlink -f $0`
|
||||
fi
|
||||
|
||||
THIS_DIR="${THIS_SCRIPT%/*}"
|
||||
BASE_DIR=$(cd $THIS_DIR/.. && pwd)
|
||||
BIN_DIR=${BASE_DIR}/bin
|
||||
|
||||
. ${BIN_DIR}/timely-balancer-env.sh
|
||||
|
||||
for i in $(seq 1 $NUM_INSTANCES); do
|
||||
PID=`jps -m | grep -E 'timely-balancer-.*-exec.jar' | grep "instance=${i}" | awk '{print $1}'`
|
||||
if [ "$PID" != "" ]; then
|
||||
echo "stopping timely-balancer instance ${i} [${PID}]"
|
||||
kill -9 $PID
|
||||
fi
|
||||
done
|
||||
Executable
+30
@@ -0,0 +1,30 @@
|
||||
#!/bin/bash
|
||||
|
||||
NUM_INSTANCES=1
|
||||
ACCUMULO_ZOOKEEPERS=localhost:2181
|
||||
HADOOP_CONF_DIR=${HADOOP_CONF_DIR}
|
||||
HADOOP_DEFAULT_FS=hdfs://localhost:8020
|
||||
TCP_BASE_PORT=4242
|
||||
HTTP_BASE_PORT=4243
|
||||
WEBSOCKET_BASE_PORT=4244
|
||||
UDP_BASE_PORT=4245
|
||||
|
||||
HOST_IP=0.0.0.0
|
||||
HOST=$(hostname -s)
|
||||
HOST_VERIFICATION_ENABLED=true
|
||||
|
||||
GENERATE_KEYS=true
|
||||
TRUST_STORE_TYPE=
|
||||
TRUST_STORE_FILE=
|
||||
TRUST_STORE_PASSWORD=
|
||||
KEY_STORE_TYPE=
|
||||
KEY_STORE_FILE=
|
||||
KEY_STORE_PASSWORD=
|
||||
AUTHORIZATION_URL=
|
||||
AUTHORIZATION_CERTIFICATE_TYPE=
|
||||
AUTHORIZATION_CERTIFICATE_FILE=
|
||||
ANONYMOUS_HTTP=true
|
||||
ANONYMOUS_WEBSOCKET=true
|
||||
|
||||
REQUIRED_ROLES=
|
||||
REQUIRED_AUTHS=
|
||||
Executable
+66
@@ -0,0 +1,66 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [[ $(uname) == "Darwin" ]]; then
|
||||
THIS_SCRIPT=$(python -c 'import os,sys; print os.path.realpath(sys.argv[1])' "$0")
|
||||
TCNATIVE_SUFFIX="jnilib"
|
||||
else
|
||||
THIS_SCRIPT=$(readlink -f "$0")
|
||||
TCNATIVE_SUFFIX="so"
|
||||
fi
|
||||
|
||||
if [ -z "$1" ]; then
|
||||
export INSTANCE="1"
|
||||
else
|
||||
export INSTANCE="$1"
|
||||
fi
|
||||
|
||||
PID=$(jps -m | grep -E 'timely-balancer-.*-exec.jar' | grep "instance=${INSTANCE}" | awk '{print $1}')
|
||||
|
||||
if [ "$PID" == "" ]; then
|
||||
|
||||
THIS_DIR="${THIS_SCRIPT%/*}"
|
||||
BASE_DIR=$(cd "$THIS_DIR"/.. && pwd)
|
||||
TMP_DIR="${BASE_DIR}/tmp"
|
||||
LIB_DIR="${BASE_DIR}/lib"
|
||||
BIN_DIR="${BASE_DIR}/bin"
|
||||
export CONF_DIR="${BASE_DIR}/conf"
|
||||
NATIVE_DIR="${BIN_DIR}/META-INF/native"
|
||||
|
||||
set -a
|
||||
. "${BIN_DIR}/timely-balancer-env.sh"
|
||||
set +a
|
||||
|
||||
# use either a value from timely-balancer-env.sh or the default
|
||||
export LOG_DIR="${TIMELY_LOG_DIR:-${BASE_DIR}/logs}"
|
||||
|
||||
if [[ -e ${TMP_DIR} ]]; then
|
||||
rm -rf "${TMP_DIR}"
|
||||
fi
|
||||
mkdir "${TMP_DIR}"
|
||||
|
||||
if [[ -e ${NATIVE_DIR} ]]; then
|
||||
rm -rf "${NATIVE_DIR}"
|
||||
fi
|
||||
mkdir -p "${NATIVE_DIR}"
|
||||
|
||||
if [[ ! -e ${LOG_DIR} ]]; then
|
||||
mkdir "${LOG_DIR}"
|
||||
fi
|
||||
|
||||
pushd "${BASE_DIR}/bin" || exit
|
||||
"${JAVA_HOME}/bin/jar xf ${LIB_DIR}/netty-tcnative-boringssl-static*.jar META-INF/native/libnetty_tcnative_linux_x86_64.${TCNATIVE_SUFFIX}"
|
||||
"${JAVA_HOME}/bin/jar xf ${LIB_DIR}/netty-all*.jar META-INF/native/libnetty_transport_native_epoll_x86_64.${TCNATIVE_SUFFIX}"
|
||||
popd || exit
|
||||
|
||||
JVM_ARGS="-Xmx2G -Xms2G -XX:NewSize=1G -XX:MaxNewSize=1G"
|
||||
JVM_ARGS="${JVM_ARGS} -Djava.library.path=${NATIVE_DIR}"
|
||||
JVM_ARGS="${JVM_ARGS} -Dlogging.config=${CONF_DIR}/log4j2.yml"
|
||||
JVM_ARGS="${JVM_ARGS} -XX:+UseG1GC -XX:+UseStringDeduplication"
|
||||
JVM_ARGS="${JVM_ARGS} -Djava.net.preferIPv4Stack=true"
|
||||
JVM_ARGS="${JVM_ARGS} -XX:+UseNUMA"
|
||||
|
||||
echo "${JAVA_HOME}/bin/java ${JVM_ARGS} -jar ${THIS_DIR}/timely-balancer-*-exec.jar --instance=${INSTANCE}"
|
||||
nohup "${JAVA_HOME}"/bin/java ${JVM_ARGS} -jar "${THIS_DIR}"/timely-balancer-*-exec.jar --instance="${INSTANCE}" >> "${LOG_DIR}/timely-balancer${INSTANCE}.out" 2>&1 &
|
||||
else
|
||||
echo "timely-balancer --instance=${INSTANCE} already running with pid ${PID}"
|
||||
fi
|
||||
@@ -0,0 +1,36 @@
|
||||
Configuration:
|
||||
status: warn
|
||||
monitorInterval: 60
|
||||
|
||||
Properties:
|
||||
Property:
|
||||
- name: logDir
|
||||
value: ${env:LOG_DIR}
|
||||
- name: PID
|
||||
value: "????"
|
||||
- name: LOG_PATTERN
|
||||
value: "%clr{%d{yyyy-MM-dd HH:mm:ss.SSS}}{faint} %clr{%5p} %clr{${sys:PID}}{magenta} %clr{---}{faint} %clr{[%15.15t]}{faint} %clr{%-40.40c{1.}}{cyan} %clr{:}{faint} %m%n%wEx"
|
||||
|
||||
Appenders:
|
||||
RollingFile:
|
||||
- name: File
|
||||
fileName: "${sys:logDir}/timely-balancer${env:INSTANCE}.log"
|
||||
filePattern: "${sys:logDir}/timely-balancer${env:INSTANCE}.log.%d{yyyy-MM-dd}-%i.gz"
|
||||
append: true
|
||||
bufferedIO: true
|
||||
bufferSize: 8192
|
||||
Policies:
|
||||
TimeBasedTriggeringPolicy:
|
||||
interval: 1
|
||||
SizeBasedTriggeringPolicy:
|
||||
size: 250MB
|
||||
DefaultRolloverStrategy:
|
||||
max: 10
|
||||
PatternLayout:
|
||||
pattern: "${LOG_PATTERN}"
|
||||
|
||||
Loggers:
|
||||
Root:
|
||||
level: info
|
||||
AppenderRef:
|
||||
- ref: File
|
||||
@@ -0,0 +1,151 @@
|
||||
timely:
|
||||
instance: ${INSTANCE}
|
||||
zookeeper:
|
||||
servers: ${ACCUMULO_ZOOKEEPERS}
|
||||
timeout: 30s
|
||||
security:
|
||||
session-max-age: 86400
|
||||
cache-expiration-minutes: 60
|
||||
cache-refresh-minutes: 5
|
||||
authorization-url: ${AUTHORIZATION_URL}
|
||||
required-roles: ${REQUIRED_ROLES}
|
||||
required-auths: ${REQUIRED_AUTHS}
|
||||
allow-anonymous-http-access: ${ANONYMOUS_HTTP:true}
|
||||
allow-anonymous-ws-access: ${ANONYMOUS_WEBSOCKET:true}
|
||||
server-ssl:
|
||||
key-store-type: ${KEY_STORE_TYPE}
|
||||
key-store-file: ${KEY_STORE_FILE}
|
||||
key-store-password: ${KEY_STORE_PASSWORD}
|
||||
use-generated-keypair: ${GENERATE_KEYS}
|
||||
ignore-ssl-handshake-errors: true
|
||||
trust-store-type: ${TRUST_STORE_TYPE}
|
||||
trust-store-file: ${TRUST_STORE_FILE}
|
||||
trust-store-password: ${TRUST_STORE_PASSWORD}
|
||||
client-ssl:
|
||||
key-store-type: ${KEY_STORE_TYPE}
|
||||
key-store-file: ${KEY_STORE_FILE}
|
||||
key-store-password: ${KEY_STORE_PASSWORD}
|
||||
trust-store-type: ${TRUST_STORE_TYPE}
|
||||
trust-store-file: ${TRUST_STORE_FILE}
|
||||
trust-store-password: ${TRUST_STORE_PASSWORD}
|
||||
hostVerificationEnabled: ${HOST_VERIFICATION_ENABLED:true}
|
||||
jwt-ssl:
|
||||
key-store-type: ${AUTHORIZATION_CERTIFICATE_TYPE}
|
||||
key-store-file: ${AUTHORIZATION_CERTIFICATE_FILE}
|
||||
key-store-password: ${AUTHORIZATION_CERTIFICATE_PASSWORD}
|
||||
cache:
|
||||
enabled: false
|
||||
metric-age-off-hours:
|
||||
default: 26
|
||||
non-cached-metrics:
|
||||
balancer:
|
||||
check-server-health-interval: 10000
|
||||
server-failures-before-down: 6
|
||||
server-successes-before-up: 1
|
||||
default-fs: ${HADOOP_DEFAULT_FS}
|
||||
fs-config-resources:
|
||||
- ${HADOOP_CONF_DIR}/core-site.xml
|
||||
- ${HADOOP_CONF_DIR}/hdfs-site.xml
|
||||
assignment-file: hdfs:///timely/balancer/assignments.csv
|
||||
server:
|
||||
ip: ${HOST_IP}
|
||||
tcp-base-port: ${TCP_BASE_PORT}
|
||||
num-tcp-pools: 1
|
||||
tcp-buffer-size: 100
|
||||
tcp-client-pool:
|
||||
min-idle-per-key: 0
|
||||
max-idle-per-key: 40
|
||||
max-total-per-key: 40
|
||||
max-total: -1
|
||||
lifo: true
|
||||
fairness: false
|
||||
max-wait-millis: -1
|
||||
min-evictable-idle-time-millis: 1800000
|
||||
evictor-shutdown-timeout-millis: 10000
|
||||
soft-min-evictable-idle-time-millis: -1
|
||||
num-tests-per-eviction-run: 3
|
||||
time-between-eviction-runs-millis: -1
|
||||
block-when-exhausted: true
|
||||
jmx-enabled: true
|
||||
test-on-borrow: false
|
||||
test-on-return: false
|
||||
udp-base-port: ${UDP_BASE_PORT}
|
||||
udp-client-pool:
|
||||
min-idle-per-key: 0
|
||||
max-idle-per-key: 40
|
||||
max-total-per-key: 40
|
||||
max-total: -1
|
||||
lifo: true
|
||||
fairness: false
|
||||
max-wait-millis: -1
|
||||
min-evictable-idle-time-millis: 1800000
|
||||
evictor-shutdown-timeout-millis: 10000
|
||||
soft-min-evictable-idle-time-millis: -1
|
||||
num-tests-per-eviction-run: 3
|
||||
time-between-eviction-runs-millis: -1
|
||||
block-when-exhausted: true
|
||||
jmx-enabled: true
|
||||
test-on-borrow: false
|
||||
test-on-return: false
|
||||
http:
|
||||
host: ${HOST}
|
||||
ip: ${HOST_IP}
|
||||
base-port: ${HTTP_BASE_PORT}
|
||||
redirect-path: /secure-me
|
||||
strict-transport-max-age: 604800
|
||||
cors:
|
||||
allow-any-origin: false
|
||||
allow-null-origin: false
|
||||
allow-credentials: true
|
||||
allowed-origins:
|
||||
allowed-methods: DELETE,GET,HEAD,OPTIONS,PUT,POST
|
||||
allowed-headers: content-type
|
||||
httpClientPool:
|
||||
min-idle-per-key: 0
|
||||
max-idle-per-key: 40
|
||||
max-total-per-key: 40
|
||||
max-total: -1
|
||||
lifo: true
|
||||
fairness: false
|
||||
max-wait-millis: -1
|
||||
min-evictable-idle-time-millis: 1800000
|
||||
evictor-shutdown-timeout-millis: 10000
|
||||
soft-min-evictable-idle-time-millis: -1
|
||||
num-tests-per-eviction-run: 3
|
||||
time-between-eviction-runs-millis: -1
|
||||
block-when-exhausted: true
|
||||
jmx-enabled: true
|
||||
test-on-borrow: false
|
||||
test-on-return: false
|
||||
websocket:
|
||||
ip: ${HOST_IP}
|
||||
base-port: ${WEBSOCKET_BASE_PORT}
|
||||
timeout: 60
|
||||
subscription-lag: 120
|
||||
scanner-batch-size: 10000
|
||||
flush-interval-seconds: 30
|
||||
scanner-read-ahead: 1
|
||||
subscription-batch-size: 5000
|
||||
incoming-buffer-size: 50000000
|
||||
ws-client-pool:
|
||||
min-idle-per-key: 0
|
||||
max-idle-per-key: 40
|
||||
max-total-per-key: 40
|
||||
max-total: -1
|
||||
lifo: true
|
||||
fairness: false
|
||||
max-wait-millis: -1
|
||||
min-evictable-idle-time-millis: 1800000
|
||||
evictor-shutdown-timeout-millis: 10000
|
||||
soft-min-evictable-idle-time-millis: -1
|
||||
num-tests-per-eviction-run: 3
|
||||
time-between-eviction-runs-millis: -1
|
||||
block-when-exhausted: true
|
||||
jmx-enabled: true
|
||||
test-on-borrow: false
|
||||
test-on-return: false
|
||||
|
||||
logging:
|
||||
level:
|
||||
ROOT: info
|
||||
timely: debug
|
||||
@@ -0,0 +1,337 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>timely</groupId>
|
||||
<artifactId>timely</artifactId>
|
||||
<version>2.1.0</version>
|
||||
</parent>
|
||||
<artifactId>timely-balancer</artifactId>
|
||||
<description>Timely Balancer</description>
|
||||
<properties>
|
||||
<start-class>timely.balancer.TimelyBalancer</start-class>
|
||||
</properties>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-annotations</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-core</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-databind</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.datatype</groupId>
|
||||
<artifactId>jackson-datatype-jdk8</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.github.ben-manes.caffeine</groupId>
|
||||
<artifactId>caffeine</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.google.flatbuffers</groupId>
|
||||
<artifactId>flatbuffers-java</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>commons-beanutils</groupId>
|
||||
<artifactId>commons-beanutils</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.github.classgraph</groupId>
|
||||
<artifactId>classgraph</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.netty</groupId>
|
||||
<artifactId>netty-all</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>net.sourceforge.javacsv</groupId>
|
||||
<artifactId>javacsv</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.accumulo</groupId>
|
||||
<artifactId>accumulo-core</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-collections4</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-lang3</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-pool2</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.curator</groupId>
|
||||
<artifactId>curator-client</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.curator</groupId>
|
||||
<artifactId>curator-framework</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.curator</groupId>
|
||||
<artifactId>curator-recipes</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.curator</groupId>
|
||||
<artifactId>curator-x-discovery</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-client</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-common</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-1.2-api</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-webflux</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-websocket</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter</artifactId>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-logging</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-log4j2</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-validation</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-web</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-context</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-starter-bootstrap</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.security</groupId>
|
||||
<artifactId>spring-security-config</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.security</groupId>
|
||||
<artifactId>spring-security-web</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>timely</groupId>
|
||||
<artifactId>timely-client</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>timely</groupId>
|
||||
<artifactId>timely-common</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.lmax</groupId>
|
||||
<artifactId>disruptor</artifactId>
|
||||
<scope>runtime</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.netty</groupId>
|
||||
<artifactId>netty-tcnative-boringssl-static</artifactId>
|
||||
<scope>runtime</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>javax.activation</groupId>
|
||||
<artifactId>activation</artifactId>
|
||||
<scope>runtime</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-api</artifactId>
|
||||
<scope>runtime</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-core</artifactId>
|
||||
<scope>runtime</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.javassist</groupId>
|
||||
<artifactId>javassist</artifactId>
|
||||
<scope>runtime</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-minicluster</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.jsoup</groupId>
|
||||
<artifactId>jsoup</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-test</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-test-autoconfigure</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>timely</groupId>
|
||||
<artifactId>timely-test</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>assemble</id>
|
||||
<goals>
|
||||
<goal>single</goal>
|
||||
</goals>
|
||||
<phase>package</phase>
|
||||
<configuration>
|
||||
<descriptors>
|
||||
<descriptor>src/main/dist/assembly.xml</descriptor>
|
||||
</descriptors>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-dependency-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>copy-dependencies</id>
|
||||
<goals>
|
||||
<goal>copy-dependencies</goal>
|
||||
</goals>
|
||||
<phase>prepare-package</phase>
|
||||
<configuration>
|
||||
<excludeArtifactIds>junit</excludeArtifactIds>
|
||||
<outputDirectory>${project.build.directory}/lib</outputDirectory>
|
||||
</configuration>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>unpack-native</id>
|
||||
<goals>
|
||||
<goal>unpack</goal>
|
||||
</goals>
|
||||
<phase>process-test-classes</phase>
|
||||
<configuration>
|
||||
<artifactItems>
|
||||
<artifactItem>
|
||||
<groupId>io.netty</groupId>
|
||||
<artifactId>netty-tcnative-boringssl-static</artifactId>
|
||||
<version>${version.netty-tcnative}</version>
|
||||
<type>jar</type>
|
||||
<overWrite>false</overWrite>
|
||||
<outputDirectory>${project.build.directory}/native</outputDirectory>
|
||||
<includes>**/*.so</includes>
|
||||
</artifactItem>
|
||||
</artifactItems>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-failsafe-plugin</artifactId>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-jar-plugin</artifactId>
|
||||
<version>3.1.2</version>
|
||||
<configuration>
|
||||
<archive>
|
||||
<manifest>
|
||||
<!--suppress UnresolvedMavenProperty, MavenModelInspection -->
|
||||
<mainClass>${start-class}</mainClass>
|
||||
<addDefaultImplementationEntries>true</addDefaultImplementationEntries>
|
||||
</manifest>
|
||||
</archive>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.codehaus.mojo</groupId>
|
||||
<artifactId>build-helper-maven-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>add-source</id>
|
||||
<goals>
|
||||
<goal>add-source</goal>
|
||||
</goals>
|
||||
<phase>generate-sources</phase>
|
||||
<configuration>
|
||||
<sources>
|
||||
<source>src/main/generated-java</source>
|
||||
</sources>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
<extensions>
|
||||
<!-- Use os-maven-plugin to initialize the "os.detected" properties -->
|
||||
<extension>
|
||||
<groupId>kr.motd.maven</groupId>
|
||||
<artifactId>os-maven-plugin</artifactId>
|
||||
</extension>
|
||||
</extensions>
|
||||
</build>
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>docker</id>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>com.spotify</groupId>
|
||||
<artifactId>docker-maven-plugin</artifactId>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
</profiles>
|
||||
</project>
|
||||
Vendored
+55
@@ -0,0 +1,55 @@
|
||||
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
|
||||
<id>dist</id>
|
||||
<formats>
|
||||
<format>tar.gz</format>
|
||||
</formats>
|
||||
<fileSets>
|
||||
<fileSet>
|
||||
<directory>${project.basedir}</directory>
|
||||
<outputDirectory></outputDirectory>
|
||||
<includes>
|
||||
<include>README*</include>
|
||||
<include>LICENSE*</include>
|
||||
<include>NOTICE*</include>
|
||||
</includes>
|
||||
</fileSet>
|
||||
<fileSet>
|
||||
<directory>${project.build.directory}/lib</directory>
|
||||
<outputDirectory>lib</outputDirectory>
|
||||
<includes>
|
||||
<include>netty-tcnative-boringssl-static*.jar</include>
|
||||
<include>netty-all*.jar</include>
|
||||
</includes>
|
||||
</fileSet>
|
||||
<fileSet>
|
||||
<directory>${project.build.directory}</directory>
|
||||
<outputDirectory>bin</outputDirectory>
|
||||
<includes>
|
||||
<include>timely-balancer-*-exec.jar</include>
|
||||
</includes>
|
||||
</fileSet>
|
||||
<fileSet>
|
||||
<directory>${project.basedir}/bin</directory>
|
||||
<outputDirectory>bin</outputDirectory>
|
||||
<includes>
|
||||
<include>**/*</include>
|
||||
</includes>
|
||||
</fileSet>
|
||||
<fileSet>
|
||||
<directory>${project.basedir}/conf</directory>
|
||||
<outputDirectory>conf</outputDirectory>
|
||||
<includes>
|
||||
<include>*</include>
|
||||
</includes>
|
||||
</fileSet>
|
||||
<fileSet>
|
||||
<directory>${project.basedir}/src/main/resources/webapp</directory>
|
||||
<outputDirectory>bin/webapp</outputDirectory>
|
||||
<includes>
|
||||
<include>*</include>
|
||||
</includes>
|
||||
</fileSet>
|
||||
</fileSets>
|
||||
</assembly>
|
||||
@@ -0,0 +1,11 @@
|
||||
FROM azul/zulu-openjdk-alpine:11
|
||||
|
||||
LABEL version=${project.version} \
|
||||
run="docker run ${docker.image.prefix}${project.artifactId}:latest" \
|
||||
description="${project.description}"
|
||||
|
||||
ADD ${project.build.finalName}-exec.jar /app.jar
|
||||
RUN apk add libc6-compat curl
|
||||
|
||||
EXPOSE 54322 54323 54324 54325
|
||||
ENTRYPOINT ["java","-jar","app.jar"]
|
||||
@@ -0,0 +1,121 @@
|
||||
package timely.balancer;
|
||||
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import com.google.common.util.concurrent.AtomicDouble;
|
||||
|
||||
public class ArrivalRate {
|
||||
|
||||
private AtomicLong shortTermArrivals = new AtomicLong(0);
|
||||
private AtomicDouble shortTermRate = new AtomicDouble(0);
|
||||
private AtomicLong shortTermLastReset = new AtomicLong(0);
|
||||
|
||||
private AtomicLong mediumTermArrivals = new AtomicLong(0);
|
||||
private AtomicDouble mediumTermRate = new AtomicDouble(0);
|
||||
private AtomicLong mediumTermLastReset = new AtomicLong(0);
|
||||
|
||||
private AtomicLong longTermArrivals = new AtomicLong(0);
|
||||
private AtomicDouble longTermRate = new AtomicDouble(0);
|
||||
private AtomicLong longTermLastReset = new AtomicLong(0);
|
||||
|
||||
public ArrivalRate(ScheduledExecutorService arrivalRateExecutor) {
|
||||
long now = System.currentTimeMillis();
|
||||
shortTermLastReset.set(now);
|
||||
mediumTermLastReset.set(now);
|
||||
longTermLastReset.set(now);
|
||||
|
||||
// 2 minute
|
||||
arrivalRateExecutor.scheduleAtFixedRate(() -> {
|
||||
resetShort();
|
||||
}, 2, 2, TimeUnit.MINUTES);
|
||||
|
||||
// 5 minute
|
||||
arrivalRateExecutor.scheduleAtFixedRate(() -> {
|
||||
resetMedium();
|
||||
}, 5, 5, TimeUnit.MINUTES);
|
||||
|
||||
// 15 minute
|
||||
arrivalRateExecutor.scheduleAtFixedRate(() -> {
|
||||
resetLong();
|
||||
}, 15, 15, TimeUnit.MINUTES);
|
||||
}
|
||||
|
||||
private void resetShort() {
|
||||
long now = System.currentTimeMillis();
|
||||
long lastReset = shortTermLastReset.get();
|
||||
if (now == lastReset) {
|
||||
shortTermRate.set(0);
|
||||
} else {
|
||||
shortTermRate.set(shortTermArrivals.get() / (((double) (now - lastReset)) / 1000));
|
||||
}
|
||||
shortTermArrivals.set(0);
|
||||
shortTermLastReset.set(now);
|
||||
}
|
||||
|
||||
private void resetMedium() {
|
||||
long now = System.currentTimeMillis();
|
||||
long lastReset = mediumTermLastReset.get();
|
||||
if (now == lastReset) {
|
||||
mediumTermRate.set(0);
|
||||
} else {
|
||||
mediumTermRate.set(mediumTermArrivals.get() / (((double) (now - lastReset)) / 1000));
|
||||
}
|
||||
mediumTermArrivals.set(0);
|
||||
mediumTermLastReset.set(now);
|
||||
}
|
||||
|
||||
private void resetLong() {
|
||||
long now = System.currentTimeMillis();
|
||||
long lastReset = longTermLastReset.get();
|
||||
if (now == lastReset) {
|
||||
longTermRate.set(0);
|
||||
} else {
|
||||
longTermRate.set(longTermArrivals.get() / (((double) (now - lastReset)) / 1000));
|
||||
}
|
||||
longTermArrivals.set(0);
|
||||
longTermLastReset.set(now);
|
||||
}
|
||||
|
||||
public void arrived() {
|
||||
shortTermArrivals.incrementAndGet();
|
||||
mediumTermArrivals.incrementAndGet();
|
||||
longTermArrivals.incrementAndGet();
|
||||
}
|
||||
|
||||
private double getShortRate(long now) {
|
||||
double rate = shortTermRate.get();
|
||||
long lastReset = shortTermLastReset.get();
|
||||
if (rate > 0 || now == lastReset) {
|
||||
return rate;
|
||||
} else {
|
||||
return shortTermArrivals.get() / (((double) (now - lastReset)) / 1000);
|
||||
}
|
||||
}
|
||||
|
||||
private double getMediumRate(long now) {
|
||||
double rate = mediumTermRate.get();
|
||||
long lastReset = mediumTermLastReset.get();
|
||||
if (rate > 0 || now == lastReset) {
|
||||
return rate;
|
||||
} else {
|
||||
return mediumTermArrivals.get() / (((double) (now - lastReset)) / 1000);
|
||||
}
|
||||
}
|
||||
|
||||
private double getLongRate(long now) {
|
||||
double rate = longTermRate.get();
|
||||
long lastReset = longTermLastReset.get();
|
||||
if (rate > 0 || now == lastReset) {
|
||||
return rate;
|
||||
} else {
|
||||
return longTermArrivals.get() / (((double) (now - lastReset)) / 1000);
|
||||
}
|
||||
}
|
||||
|
||||
public double getRate() {
|
||||
long now = System.currentTimeMillis();
|
||||
return 0.10 * getLongRate(now) + 0.10 * getMediumRate(now) + 0.80 * getShortRate(now);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,490 @@
|
||||
package timely.balancer;
|
||||
|
||||
import static org.apache.accumulo.core.conf.ConfigurationTypeHelper.getTimeInMillis;
|
||||
import static timely.Constants.NON_CACHED_METRICS;
|
||||
import static timely.Constants.NON_CACHED_METRICS_LOCK_PATH;
|
||||
import static timely.Constants.SERVICE_DISCOVERY_PATH;
|
||||
|
||||
import java.net.InetSocketAddress;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import javax.net.ssl.SSLContext;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.curator.RetryPolicy;
|
||||
import org.apache.curator.framework.CuratorFramework;
|
||||
import org.apache.curator.framework.CuratorFrameworkFactory;
|
||||
import org.apache.curator.retry.RetryForever;
|
||||
import org.apache.zookeeper.CreateMode;
|
||||
import org.apache.zookeeper.data.Stat;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
|
||||
import io.netty.bootstrap.AbstractBootstrap;
|
||||
import io.netty.bootstrap.Bootstrap;
|
||||
import io.netty.bootstrap.ServerBootstrap;
|
||||
import io.netty.buffer.PooledByteBufAllocator;
|
||||
import io.netty.channel.Channel;
|
||||
import io.netty.channel.ChannelFuture;
|
||||
import io.netty.channel.ChannelHandler;
|
||||
import io.netty.channel.ChannelInitializer;
|
||||
import io.netty.channel.ChannelOption;
|
||||
import io.netty.channel.EventLoopGroup;
|
||||
import io.netty.channel.epoll.EpollChannelOption;
|
||||
import io.netty.channel.epoll.EpollDatagramChannel;
|
||||
import io.netty.channel.epoll.EpollEventLoopGroup;
|
||||
import io.netty.channel.epoll.EpollServerSocketChannel;
|
||||
import io.netty.channel.nio.NioEventLoopGroup;
|
||||
import io.netty.channel.socket.DatagramChannel;
|
||||
import io.netty.channel.socket.ServerSocketChannel;
|
||||
import io.netty.channel.socket.SocketChannel;
|
||||
import io.netty.channel.socket.nio.NioDatagramChannel;
|
||||
import io.netty.channel.socket.nio.NioServerSocketChannel;
|
||||
import io.netty.handler.codec.DelimiterBasedFrameDecoder;
|
||||
import io.netty.handler.codec.Delimiters;
|
||||
import io.netty.handler.codec.http.HttpContentCompressor;
|
||||
import io.netty.handler.codec.http.HttpContentDecompressor;
|
||||
import io.netty.handler.codec.http.HttpObjectAggregator;
|
||||
import io.netty.handler.codec.http.HttpRequestDecoder;
|
||||
import io.netty.handler.codec.http.HttpResponseEncoder;
|
||||
import io.netty.handler.codec.http.HttpServerCodec;
|
||||
import io.netty.handler.codec.http.websocketx.WebSocketServerProtocolHandler;
|
||||
import io.netty.handler.logging.LoggingHandler;
|
||||
import io.netty.handler.ssl.OpenSslServerContext;
|
||||
import io.netty.handler.ssl.OpenSslServerSessionContext;
|
||||
import io.netty.handler.ssl.SslContext;
|
||||
import io.netty.handler.stream.ChunkedWriteHandler;
|
||||
import io.netty.handler.timeout.IdleStateHandler;
|
||||
import io.netty.util.NettyRuntime;
|
||||
import io.netty.util.internal.SystemPropertyUtil;
|
||||
import timely.auth.JWTTokenHandler;
|
||||
import timely.balancer.configuration.BalancerHttpProperties;
|
||||
import timely.balancer.configuration.BalancerProperties;
|
||||
import timely.balancer.configuration.BalancerServerProperties;
|
||||
import timely.balancer.configuration.BalancerWebsocketProperties;
|
||||
import timely.balancer.connection.http.HttpClientPool;
|
||||
import timely.balancer.connection.tcp.TcpClientPool;
|
||||
import timely.balancer.connection.udp.UdpClientPool;
|
||||
import timely.balancer.connection.ws.WsClientPool;
|
||||
import timely.balancer.netty.http.HttpRelayHandler;
|
||||
import timely.balancer.netty.tcp.TcpRelayHandler;
|
||||
import timely.balancer.netty.udp.UdpRelayHandler;
|
||||
import timely.balancer.netty.ws.WsRelayHandler;
|
||||
import timely.common.component.AuthenticationService;
|
||||
import timely.common.configuration.SecurityProperties;
|
||||
import timely.common.configuration.SslClientProperties;
|
||||
import timely.common.configuration.SslServerProperties;
|
||||
import timely.common.configuration.ZookeeperProperties;
|
||||
import timely.netty.http.HttpStaticFileServerHandler;
|
||||
import timely.netty.http.NonSslRedirectHandler;
|
||||
import timely.netty.http.TimelyExceptionHandler;
|
||||
import timely.netty.http.auth.X509LoginRequestHandler;
|
||||
import timely.netty.tcp.MetricsBufferDecoder;
|
||||
import timely.netty.tcp.TcpDecoder;
|
||||
import timely.netty.tcp.TcpVersionHandler;
|
||||
import timely.netty.udp.UdpDecoder;
|
||||
import timely.netty.udp.UdpPacketToByteBuf;
|
||||
import timely.netty.websocket.WebSocketFullRequestHandler;
|
||||
import timely.netty.websocket.WebSocketRequestDecoder;
|
||||
import timely.netty.websocket.subscription.WSTimelyExceptionHandler;
|
||||
|
||||
public class Balancer {
|
||||
|
||||
final public static String LEADER_LATCH_PATH = "/timely/balancer/leader";
|
||||
final public static String ASSIGNMENTS_LOCK_PATH = "/timely/balancer/assignments/lock";
|
||||
final public static String ASSIGNMENTS_LAST_UPDATED_PATH = "/timely/balancer/assignments/lastUpdated";
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(Balancer.class);
|
||||
private static final String EPOLL_MIN_VERSION = "2.7";
|
||||
private static final String OS_NAME = "os.name";
|
||||
private static final String OS_VERSION = "os.version";
|
||||
private static final String WS_PATH = "/websocket";
|
||||
private BalancerProperties balancerProperties;
|
||||
private SecurityProperties securityProperties;
|
||||
private SslServerProperties sslServerProperties;
|
||||
private SslClientProperties sslClientProperties;
|
||||
private ZookeeperProperties zookeeperProperties;
|
||||
private BalancerServerProperties balancerServerProperties;
|
||||
private BalancerHttpProperties balancerHttpProperties;
|
||||
private BalancerWebsocketProperties balancerWebsocketProperties;
|
||||
private ApplicationContext applicationContext;
|
||||
private AuthenticationService authenticationService;
|
||||
private CuratorFramework curatorFramework;
|
||||
private int quietPeriod;
|
||||
private SslContext serverSslContext;
|
||||
private SSLContext clientSSLContext;
|
||||
private EventLoopGroup tcpWorkerGroup = null;
|
||||
private EventLoopGroup tcpBossGroup = null;
|
||||
private EventLoopGroup httpWorkerGroup = null;
|
||||
private EventLoopGroup httpBossGroup = null;
|
||||
private EventLoopGroup wsWorkerGroup = null;
|
||||
private EventLoopGroup wsBossGroup = null;
|
||||
private EventLoopGroup udpBossGroup = null;
|
||||
private EventLoopGroup udpWorkerGroup = null;
|
||||
private MetricResolver metricResolver;
|
||||
protected Channel tcpChannelHandle = null;
|
||||
protected Channel httpChannelHandle = null;
|
||||
protected Channel wsChannelHandle = null;
|
||||
protected List<Channel> udpChannelHandleList = new ArrayList<>();
|
||||
protected int DEFAULT_EVENT_LOOP_THREADS;
|
||||
|
||||
private String[] zkPaths = new String[] {LEADER_LATCH_PATH, ASSIGNMENTS_LAST_UPDATED_PATH, ASSIGNMENTS_LOCK_PATH, SERVICE_DISCOVERY_PATH,
|
||||
NON_CACHED_METRICS, NON_CACHED_METRICS_LOCK_PATH};
|
||||
|
||||
private static boolean useEpoll() {
|
||||
final String osName = SystemPropertyUtil.get(OS_NAME).toLowerCase().trim();
|
||||
final String osVersion = SystemPropertyUtil.get(OS_VERSION).toLowerCase();
|
||||
// split at periods, keep the first two, join with a period
|
||||
final String majMinVers = StringUtils.join(Arrays.stream(osVersion.split("\\.")).limit(2).collect(Collectors.toList()), ".");
|
||||
if (osName.startsWith("linux")) {
|
||||
try {
|
||||
Runtime.Version currentVersion = Runtime.Version.parse(majMinVers);
|
||||
Runtime.Version epollMinVersion = Runtime.Version.parse(EPOLL_MIN_VERSION);
|
||||
return currentVersion.compareTo(epollMinVersion) > 0;
|
||||
} catch (Exception e) {
|
||||
log.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public Balancer(ApplicationContext applicationContext, SslContext serverSslContext, SSLContext clientSSLContext,
|
||||
AuthenticationService authenticationService, MetricResolver metricResolver, BalancerProperties balancerProperties,
|
||||
ZookeeperProperties zookeeperProperties, SecurityProperties securityProperties, SslServerProperties sslServerProperties,
|
||||
SslClientProperties sslClientProperties, BalancerServerProperties balancerServerProperties, BalancerHttpProperties balancerHttpProperties,
|
||||
BalancerWebsocketProperties balancerWebsocketProperties) {
|
||||
this.serverSslContext = serverSslContext;
|
||||
this.clientSSLContext = clientSSLContext;
|
||||
this.quietPeriod = balancerServerProperties.getShutdownQuietPeriod();
|
||||
this.applicationContext = applicationContext;
|
||||
this.authenticationService = authenticationService;
|
||||
this.metricResolver = metricResolver;
|
||||
this.balancerProperties = balancerProperties;
|
||||
this.zookeeperProperties = zookeeperProperties;
|
||||
this.securityProperties = securityProperties;
|
||||
this.sslServerProperties = sslServerProperties;
|
||||
this.sslClientProperties = sslClientProperties;
|
||||
this.balancerServerProperties = balancerServerProperties;
|
||||
this.balancerHttpProperties = balancerHttpProperties;
|
||||
this.balancerWebsocketProperties = balancerWebsocketProperties;
|
||||
DEFAULT_EVENT_LOOP_THREADS = Math.max(1, SystemPropertyUtil.getInt("io.netty.eventLoopThreads", NettyRuntime.availableProcessors() * 2));
|
||||
}
|
||||
|
||||
public void start() {
|
||||
log.info("Starting {}", this.getClass().getSimpleName());
|
||||
try {
|
||||
// start curator framework
|
||||
RetryPolicy retryPolicy = new RetryForever(1000);
|
||||
int timeout = Long.valueOf(getTimeInMillis(zookeeperProperties.getTimeout())).intValue();
|
||||
curatorFramework = CuratorFrameworkFactory.newClient(zookeeperProperties.getServers(), timeout, 10000, retryPolicy);
|
||||
curatorFramework.start();
|
||||
ensureZkPaths(curatorFramework, zkPaths);
|
||||
|
||||
JWTTokenHandler.init(securityProperties, null);
|
||||
final boolean useEpoll = useEpoll();
|
||||
Class<? extends ServerSocketChannel> channelClass;
|
||||
Class<? extends Channel> datagramChannelClass;
|
||||
if (useEpoll) {
|
||||
tcpWorkerGroup = new EpollEventLoopGroup();
|
||||
tcpBossGroup = new EpollEventLoopGroup();
|
||||
httpWorkerGroup = new EpollEventLoopGroup();
|
||||
httpBossGroup = new EpollEventLoopGroup();
|
||||
wsWorkerGroup = new EpollEventLoopGroup();
|
||||
wsBossGroup = new EpollEventLoopGroup();
|
||||
udpWorkerGroup = new EpollEventLoopGroup();
|
||||
udpBossGroup = new EpollEventLoopGroup();
|
||||
channelClass = EpollServerSocketChannel.class;
|
||||
datagramChannelClass = EpollDatagramChannel.class;
|
||||
} else {
|
||||
tcpWorkerGroup = new NioEventLoopGroup();
|
||||
tcpBossGroup = new NioEventLoopGroup();
|
||||
httpWorkerGroup = new NioEventLoopGroup();
|
||||
httpBossGroup = new NioEventLoopGroup();
|
||||
wsWorkerGroup = new NioEventLoopGroup();
|
||||
wsBossGroup = new NioEventLoopGroup();
|
||||
udpWorkerGroup = new NioEventLoopGroup();
|
||||
udpBossGroup = new NioEventLoopGroup();
|
||||
channelClass = NioServerSocketChannel.class;
|
||||
datagramChannelClass = NioDatagramChannel.class;
|
||||
}
|
||||
log.info("Using channel class {}", channelClass.getSimpleName());
|
||||
|
||||
List<TcpClientPool> tcpClientPools = new ArrayList<>();
|
||||
for (int x = 0; x < balancerServerProperties.getNumTcpPools(); x++) {
|
||||
tcpClientPools.add(new TcpClientPool(balancerServerProperties));
|
||||
}
|
||||
|
||||
final ServerBootstrap tcpServer = new ServerBootstrap();
|
||||
tcpServer.group(tcpBossGroup, tcpWorkerGroup);
|
||||
tcpServer.channel(channelClass);
|
||||
tcpServer.handler(new LoggingHandler());
|
||||
tcpServer.childHandler(setupTcpChannel(metricResolver, tcpClientPools));
|
||||
tcpServer.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);
|
||||
tcpServer.option(ChannelOption.SO_BACKLOG, 128);
|
||||
tcpServer.childOption(ChannelOption.SO_KEEPALIVE, true);
|
||||
final int tcpPort = balancerServerProperties.getTcpPort();
|
||||
final String tcpIp = balancerServerProperties.getIp();
|
||||
tcpChannelHandle = bind(tcpServer, tcpIp, tcpPort);
|
||||
final String tcpAddress = ((InetSocketAddress) tcpChannelHandle.localAddress()).getAddress().getHostAddress();
|
||||
|
||||
final int httpPort = balancerHttpProperties.getPort();
|
||||
final String httpIp = balancerHttpProperties.getIp();
|
||||
|
||||
if (serverSslContext instanceof OpenSslServerContext) {
|
||||
OpenSslServerContext openssl = (OpenSslServerContext) serverSslContext;
|
||||
String application = "Timely_" + httpPort;
|
||||
OpenSslServerSessionContext opensslCtx = openssl.sessionContext();
|
||||
opensslCtx.setSessionCacheEnabled(true);
|
||||
opensslCtx.setSessionCacheSize(128);
|
||||
opensslCtx.setSessionIdContext(application.getBytes(StandardCharsets.UTF_8));
|
||||
opensslCtx.setSessionTimeout(securityProperties.getSessionMaxAge());
|
||||
}
|
||||
final ServerBootstrap httpServer = new ServerBootstrap();
|
||||
httpServer.group(httpBossGroup, httpWorkerGroup);
|
||||
httpServer.channel(channelClass);
|
||||
httpServer.handler(new LoggingHandler());
|
||||
HttpClientPool httpClientPool = new HttpClientPool(balancerHttpProperties, clientSSLContext, sslClientProperties);
|
||||
httpServer.childHandler(setupHttpChannel(balancerHttpProperties, sslServerProperties, serverSslContext, metricResolver, httpClientPool));
|
||||
httpServer.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);
|
||||
httpServer.option(ChannelOption.SO_BACKLOG, 128);
|
||||
httpServer.childOption(ChannelOption.SO_KEEPALIVE, true);
|
||||
httpChannelHandle = bind(httpServer, httpIp, httpPort);
|
||||
final String httpAddress = ((InetSocketAddress) httpChannelHandle.localAddress()).getAddress().getHostAddress();
|
||||
|
||||
final int wsPort = balancerWebsocketProperties.getPort();
|
||||
final String wsIp = balancerWebsocketProperties.getIp();
|
||||
final ServerBootstrap wsServer = new ServerBootstrap();
|
||||
wsServer.group(wsBossGroup, wsWorkerGroup);
|
||||
wsServer.channel(channelClass);
|
||||
wsServer.handler(new LoggingHandler());
|
||||
WsClientPool wsClientPool = new WsClientPool(balancerProperties, balancerWebsocketProperties, sslClientProperties, clientSSLContext);
|
||||
wsServer.childHandler(setupWSChannel(balancerWebsocketProperties, securityProperties, serverSslContext, metricResolver, wsClientPool));
|
||||
wsServer.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);
|
||||
wsServer.option(ChannelOption.SO_BACKLOG, 128);
|
||||
wsServer.childOption(ChannelOption.SO_KEEPALIVE, true);
|
||||
wsChannelHandle = bind(wsServer, wsIp, wsPort);
|
||||
final String wsAddress = ((InetSocketAddress) wsChannelHandle.localAddress()).getAddress().getHostAddress();
|
||||
|
||||
final int udpPort = balancerServerProperties.getUdpPort();
|
||||
final String udpIp = balancerServerProperties.getIp();
|
||||
UdpClientPool udpClientPool = new UdpClientPool(balancerServerProperties);
|
||||
for (int n = 0; n < DEFAULT_EVENT_LOOP_THREADS; n++) {
|
||||
final Bootstrap udpServer = new Bootstrap();
|
||||
udpServer.group(udpBossGroup);
|
||||
udpServer.channel(datagramChannelClass);
|
||||
udpServer.handler(setupUdpChannel(metricResolver, udpClientPool));
|
||||
udpServer.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);
|
||||
udpServer.option(EpollChannelOption.SO_REUSEADDR, true);
|
||||
udpServer.option(EpollChannelOption.SO_REUSEPORT, true);
|
||||
udpChannelHandleList.add(bind(udpServer, udpIp, udpPort));
|
||||
}
|
||||
log.info("Balancer started. Listening on {}:{} for TCP traffic, {}:{} for HTTP traffic, {}:{} for WebSocket traffic, and {}:{} for UDP traffic",
|
||||
tcpAddress, tcpPort, httpAddress, httpPort, wsAddress, wsPort, wsAddress, udpPort);
|
||||
} catch (Exception e) {
|
||||
log.error(e.getMessage(), e);
|
||||
SpringApplication.exit(applicationContext, () -> 0);
|
||||
}
|
||||
}
|
||||
|
||||
protected Channel bind(AbstractBootstrap server, String ip, int port) {
|
||||
Channel channel = null;
|
||||
try {
|
||||
channel = server.bind(ip, port).sync().channel();
|
||||
} catch (InterruptedException e) {
|
||||
log.error("Failed to bind to {}:{} -- {}", ip, port, e.getMessage());
|
||||
}
|
||||
if (channel == null) {
|
||||
throw new IllegalStateException("Failed to bind to port:" + ip + ":" + port);
|
||||
}
|
||||
return channel;
|
||||
}
|
||||
|
||||
public void shutdown() {
|
||||
List<ChannelFuture> channelFutures = new ArrayList<>();
|
||||
|
||||
if (tcpChannelHandle != null) {
|
||||
log.info("Closing tcpChannelHandle");
|
||||
channelFutures.add(tcpChannelHandle.close());
|
||||
}
|
||||
|
||||
if (httpChannelHandle != null) {
|
||||
log.info("Closing httpChannelHandle");
|
||||
channelFutures.add(httpChannelHandle.close());
|
||||
}
|
||||
|
||||
if (wsChannelHandle != null) {
|
||||
log.info("Closing wsChannelHandle");
|
||||
channelFutures.add(wsChannelHandle.close());
|
||||
}
|
||||
|
||||
int udpChannel = 1;
|
||||
for (Channel c : udpChannelHandleList) {
|
||||
log.info("Closing udpChannelHandle #" + udpChannel++);
|
||||
channelFutures.add(c.close());
|
||||
}
|
||||
|
||||
// wait for the channels to shutdown
|
||||
channelFutures.forEach(f -> {
|
||||
try {
|
||||
f.get();
|
||||
} catch (final Exception e) {
|
||||
log.error("Channel:" + f.channel().config() + " -> " + e.getMessage(), e);
|
||||
}
|
||||
});
|
||||
|
||||
List<Future<?>> groupFutures = new ArrayList<>();
|
||||
|
||||
if (tcpBossGroup != null) {
|
||||
log.info("Shutting down tcpBossGroup");
|
||||
groupFutures.add(tcpBossGroup.shutdownGracefully(quietPeriod, 10, TimeUnit.SECONDS));
|
||||
}
|
||||
|
||||
if (tcpWorkerGroup != null) {
|
||||
log.info("Shutting down tcpWorkerGroup");
|
||||
groupFutures.add(tcpWorkerGroup.shutdownGracefully(quietPeriod, 10, TimeUnit.SECONDS));
|
||||
}
|
||||
|
||||
if (httpBossGroup != null) {
|
||||
log.info("Shutting down httpBossGroup");
|
||||
groupFutures.add(httpBossGroup.shutdownGracefully(quietPeriod, 10, TimeUnit.SECONDS));
|
||||
}
|
||||
|
||||
if (httpWorkerGroup != null) {
|
||||
log.info("Shutting down httpWorkerGroup");
|
||||
groupFutures.add(httpWorkerGroup.shutdownGracefully(quietPeriod, 10, TimeUnit.SECONDS));
|
||||
}
|
||||
|
||||
if (wsBossGroup != null) {
|
||||
log.info("Shutting down wsBossGroup");
|
||||
groupFutures.add(wsBossGroup.shutdownGracefully(quietPeriod, 10, TimeUnit.SECONDS));
|
||||
}
|
||||
|
||||
if (wsWorkerGroup != null) {
|
||||
log.info("Shutting down wsWorkerGroup");
|
||||
groupFutures.add(wsWorkerGroup.shutdownGracefully(quietPeriod, 10, TimeUnit.SECONDS));
|
||||
}
|
||||
|
||||
if (udpBossGroup != null) {
|
||||
log.info("Shutting down udpBossGroup");
|
||||
groupFutures.add(udpBossGroup.shutdownGracefully(quietPeriod, 10, TimeUnit.SECONDS));
|
||||
}
|
||||
|
||||
if (udpWorkerGroup != null) {
|
||||
log.info("Shutting down udpWorkerGroup");
|
||||
groupFutures.add(udpWorkerGroup.shutdownGracefully(quietPeriod, 10, TimeUnit.SECONDS));
|
||||
}
|
||||
|
||||
groupFutures.parallelStream().forEach(f -> {
|
||||
try {
|
||||
f.get();
|
||||
} catch (final Exception e) {
|
||||
log.error("Group:" + f.toString() + " -> " + e.getMessage(), e);
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
log.info("Closing WebSocketRequestDecoder");
|
||||
WebSocketRequestDecoder.close();
|
||||
} catch (Exception e) {
|
||||
log.error("Error closing WebSocketRequestDecoder during shutdown", e);
|
||||
}
|
||||
log.info("{} shut down.", this.getClass().getSimpleName());
|
||||
}
|
||||
|
||||
private void ensureZkPaths(CuratorFramework curatorFramework, String[] paths) {
|
||||
for (String s : paths) {
|
||||
try {
|
||||
Stat stat = curatorFramework.checkExists().forPath(s);
|
||||
if (stat == null) {
|
||||
curatorFramework.create().creatingParentContainersIfNeeded().withMode(CreateMode.PERSISTENT).forPath(s);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.info(e.getMessage());
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
protected ChannelHandler setupHttpChannel(BalancerHttpProperties balancerHttpProperties, SslServerProperties sslServerProperties, SslContext sslCtx,
|
||||
MetricResolver metricResolver, HttpClientPool httpClientPool) {
|
||||
|
||||
return new ChannelInitializer<SocketChannel>() {
|
||||
|
||||
@Override
|
||||
protected void initChannel(SocketChannel ch) {
|
||||
|
||||
ch.pipeline().addLast("ssl", new NonSslRedirectHandler(balancerHttpProperties, sslCtx));
|
||||
ch.pipeline().addLast("encoder", new HttpResponseEncoder());
|
||||
ch.pipeline().addLast("decoder", new HttpRequestDecoder(4096, 32768, 8192, true, 128));
|
||||
ch.pipeline().addLast("compressor", new HttpContentCompressor());
|
||||
ch.pipeline().addLast("decompressor", new HttpContentDecompressor());
|
||||
ch.pipeline().addLast("aggregator", new HttpObjectAggregator(65536));
|
||||
ch.pipeline().addLast("chunker", new ChunkedWriteHandler());
|
||||
ch.pipeline().addLast("queryDecoder",
|
||||
new timely.netty.http.HttpRequestDecoder(authenticationService, securityProperties, balancerHttpProperties));
|
||||
ch.pipeline().addLast("fileServer",
|
||||
new HttpStaticFileServerHandler().setIgnoreSslHandshakeErrors(sslServerProperties.isIgnoreSslHandshakeErrors()));
|
||||
ch.pipeline().addLast("login", new X509LoginRequestHandler(authenticationService, securityProperties, balancerHttpProperties));
|
||||
ch.pipeline().addLast("httpRelay", new HttpRelayHandler(metricResolver, httpClientPool));
|
||||
ch.pipeline().addLast("error", new TimelyExceptionHandler().setIgnoreSslHandshakeErrors(sslServerProperties.isIgnoreSslHandshakeErrors()));
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
protected ChannelHandler setupUdpChannel(MetricResolver metricResolver, UdpClientPool udpClientPool) {
|
||||
return new ChannelInitializer<DatagramChannel>() {
|
||||
|
||||
@Override
|
||||
protected void initChannel(DatagramChannel ch) throws Exception {
|
||||
ch.pipeline().addLast("logger", new LoggingHandler());
|
||||
ch.pipeline().addLast("packetDecoder", new UdpPacketToByteBuf());
|
||||
ch.pipeline().addLast("buffer", new MetricsBufferDecoder());
|
||||
ch.pipeline().addLast("frame", new DelimiterBasedFrameDecoder(65536, true, Delimiters.lineDelimiter()));
|
||||
ch.pipeline().addLast("putDecoder", new UdpDecoder());
|
||||
ch.pipeline().addLast("udpRelayHandler", new UdpRelayHandler(metricResolver, udpClientPool));
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
protected ChannelHandler setupTcpChannel(MetricResolver metricResolver, List<TcpClientPool> tcpClientPools) {
|
||||
return new ChannelInitializer<SocketChannel>() {
|
||||
|
||||
@Override
|
||||
protected void initChannel(SocketChannel ch) throws Exception {
|
||||
ch.pipeline().addLast("buffer", new MetricsBufferDecoder());
|
||||
ch.pipeline().addLast("frame", new DelimiterBasedFrameDecoder(65536, true, Delimiters.lineDelimiter()));
|
||||
ch.pipeline().addLast("putDecoder", new TcpDecoder());
|
||||
ch.pipeline().addLast("tcpRelayHandler", new TcpRelayHandler(metricResolver, tcpClientPools));
|
||||
ch.pipeline().addLast("versionHandler", new TcpVersionHandler());
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
protected ChannelHandler setupWSChannel(BalancerWebsocketProperties balancerWebsocketProperties, SecurityProperties securityProperties, SslContext sslCtx,
|
||||
MetricResolver metricResolver, WsClientPool wsClientPool) {
|
||||
return new ChannelInitializer<SocketChannel>() {
|
||||
|
||||
@Override
|
||||
protected void initChannel(SocketChannel ch) throws Exception {
|
||||
ch.pipeline().addLast("ssl", sslCtx.newHandler(ch.alloc()));
|
||||
ch.pipeline().addLast("httpServer", new HttpServerCodec());
|
||||
ch.pipeline().addLast("aggregator", new HttpObjectAggregator(65536));
|
||||
ch.pipeline().addLast("sessionExtractor", new WebSocketFullRequestHandler(authenticationService));
|
||||
|
||||
ch.pipeline().addLast("idle-handler", new IdleStateHandler(balancerWebsocketProperties.getTimeout(), 0, 0));
|
||||
ch.pipeline().addLast("ws-protocol", new WebSocketServerProtocolHandler(WS_PATH, null, true, 65536, false, true));
|
||||
ch.pipeline().addLast("wsDecoder", new WebSocketRequestDecoder(authenticationService, securityProperties));
|
||||
ch.pipeline().addLast("httpRelay", new WsRelayHandler(balancerWebsocketProperties, metricResolver, wsClientPool));
|
||||
ch.pipeline().addLast("error", new WSTimelyExceptionHandler());
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package timely.balancer;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import timely.balancer.configuration.BalancerProperties;
|
||||
import timely.balancer.configuration.BalancerServerProperties;
|
||||
import timely.balancer.connection.TimelyBalancedHost;
|
||||
import timely.balancer.connection.tcp.TcpClientPool;
|
||||
import timely.balancer.healthcheck.TCPServerHealthCheck;
|
||||
|
||||
public class HealthChecker {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(HealthChecker.class);
|
||||
private ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1);
|
||||
private TCPServerHealthCheck tcpServerHealthCheck;
|
||||
private BalancerProperties balancerProperties;
|
||||
private BalancerServerProperties balancerServerProperties;
|
||||
protected List<TimelyBalancedHost> timelyHosts = new ArrayList<>();
|
||||
|
||||
public HealthChecker(BalancerProperties balancerProperties, BalancerServerProperties balancerServerProperties) {
|
||||
this.balancerProperties = balancerProperties;
|
||||
this.balancerServerProperties = balancerServerProperties;
|
||||
}
|
||||
|
||||
public void start() {
|
||||
TcpClientPool tcpClientPool = new TcpClientPool(balancerServerProperties);
|
||||
this.tcpServerHealthCheck = new TCPServerHealthCheck(tcpClientPool);
|
||||
this.executorService.scheduleAtFixedRate(() -> {
|
||||
synchronized (timelyHosts) {
|
||||
for (TimelyBalancedHost h : timelyHosts) {
|
||||
try {
|
||||
if (tcpServerHealthCheck.isServerHealthy(h)) {
|
||||
h.reportSuccess();
|
||||
} else {
|
||||
h.reportFailure();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error(e.getMessage(), e);
|
||||
h.reportFailure();
|
||||
}
|
||||
}
|
||||
}
|
||||
}, 0, balancerProperties.getCheckServerHealthInterval(), TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
public void shutdown() {
|
||||
this.executorService.shutdown();
|
||||
try {
|
||||
this.executorService.awaitTermination(5, TimeUnit.SECONDS);
|
||||
} catch (InterruptedException e) {
|
||||
|
||||
} finally {
|
||||
if (!this.executorService.isTerminated()) {
|
||||
this.executorService.shutdownNow();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void setTimelyHosts(List<TimelyBalancedHost> timelyHosts) {
|
||||
synchronized (timelyHosts) {
|
||||
this.timelyHosts.clear();
|
||||
this.timelyHosts.addAll(timelyHosts);
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,14 @@
|
||||
package timely.balancer;
|
||||
|
||||
import org.springframework.boot.WebApplicationType;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.autoconfigure.web.servlet.error.ErrorMvcAutoConfiguration;
|
||||
import org.springframework.boot.builder.SpringApplicationBuilder;
|
||||
|
||||
@SpringBootApplication(scanBasePackages = {"timely.balancer", "timely.common"}, exclude = {ErrorMvcAutoConfiguration.class})
|
||||
public class TimelyBalancer {
|
||||
|
||||
public static void main(String[] args) {
|
||||
new SpringApplicationBuilder(TimelyBalancer.class).web(WebApplicationType.NONE).run(args);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package timely.balancer.configuration;
|
||||
|
||||
import javax.net.ssl.SSLContext;
|
||||
|
||||
import org.apache.curator.framework.CuratorFramework;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
import io.netty.handler.ssl.SslContext;
|
||||
import timely.balancer.Balancer;
|
||||
import timely.balancer.HealthChecker;
|
||||
import timely.balancer.MetricResolver;
|
||||
import timely.common.component.AuthenticationService;
|
||||
import timely.common.configuration.CacheProperties;
|
||||
import timely.common.configuration.SecurityProperties;
|
||||
import timely.common.configuration.SslClientProperties;
|
||||
import timely.common.configuration.SslServerProperties;
|
||||
import timely.common.configuration.ZookeeperProperties;
|
||||
|
||||
@Configuration
|
||||
@EnableConfigurationProperties({BalancerProperties.class, BalancerServerProperties.class, BalancerWebsocketProperties.class, BalancerHttpProperties.class})
|
||||
public class BalancerConfiguration {
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean
|
||||
public Balancer balancer(ApplicationContext applicationContext, @Qualifier("nettySslContext") SslContext serverSslContext,
|
||||
@Qualifier("outboundJDKSslContext") SSLContext clientSSLContext, AuthenticationService authenticationService, MetricResolver metricResolver,
|
||||
BalancerProperties balancerProperties, ZookeeperProperties zookeeperProperties, SecurityProperties securityProperties,
|
||||
SslServerProperties sslServerProperties, SslClientProperties sslClientProperties, BalancerServerProperties balancerServerProperties,
|
||||
BalancerHttpProperties balancerHttpProperties, BalancerWebsocketProperties balancerWebsocketProperties) {
|
||||
Balancer balancer = new Balancer(applicationContext, serverSslContext, clientSSLContext, authenticationService, metricResolver, balancerProperties,
|
||||
zookeeperProperties, securityProperties, sslServerProperties, sslClientProperties, balancerServerProperties, balancerHttpProperties,
|
||||
balancerWebsocketProperties);
|
||||
balancer.start();
|
||||
return balancer;
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean
|
||||
public HealthChecker healthChecker(BalancerProperties balancerProperties, BalancerServerProperties balancerServerProperties) {
|
||||
HealthChecker healthChecker = new HealthChecker(balancerProperties, balancerServerProperties);
|
||||
healthChecker.start();
|
||||
return healthChecker;
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean
|
||||
public MetricResolver metricResolver(CuratorFramework curatorFramework, BalancerProperties balancerProperties, CacheProperties cacheProperties,
|
||||
HealthChecker healthChecker) {
|
||||
MetricResolver metricResolver = new MetricResolver(curatorFramework, balancerProperties, cacheProperties, healthChecker);
|
||||
metricResolver.start();
|
||||
return metricResolver;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package timely.balancer.configuration;
|
||||
|
||||
import javax.validation.Valid;
|
||||
|
||||
import org.apache.commons.pool2.impl.GenericKeyedObjectPoolConfig;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.boot.context.properties.NestedConfigurationProperty;
|
||||
import org.springframework.cloud.context.config.annotation.RefreshScope;
|
||||
|
||||
import timely.common.configuration.HttpProperties;
|
||||
|
||||
@RefreshScope
|
||||
@ConfigurationProperties(prefix = "timely.balancer.http")
|
||||
public class BalancerHttpProperties extends HttpProperties {
|
||||
|
||||
@Valid
|
||||
@NestedConfigurationProperty
|
||||
private GenericKeyedObjectPoolConfig<CloseableHttpClient> httpClientPool = new GenericKeyedObjectPoolConfig<>();
|
||||
|
||||
public GenericKeyedObjectPoolConfig<CloseableHttpClient> getHttpClientPool() {
|
||||
return httpClientPool;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package timely.balancer.configuration;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.validation.annotation.Validated;
|
||||
|
||||
@Validated
|
||||
@Component
|
||||
@ConfigurationProperties(prefix = "timely.balancer")
|
||||
public class BalancerProperties {
|
||||
|
||||
private boolean loginRequired = false;
|
||||
|
||||
private String assignmentFile;
|
||||
|
||||
private String defaultFs;
|
||||
|
||||
private List<String> fsConfigResources;
|
||||
|
||||
private Double controlBandPercentage = 0.05;
|
||||
|
||||
private long checkServerHealthInterval = 10000;
|
||||
|
||||
private int serverFailuresBeforeDown = 3;
|
||||
|
||||
private int serverSuccessesBeforeUp = 3;
|
||||
|
||||
public void setLoginRequired(boolean loginRequired) {
|
||||
this.loginRequired = loginRequired;
|
||||
}
|
||||
|
||||
public boolean isLoginRequired() {
|
||||
return loginRequired;
|
||||
}
|
||||
|
||||
public String getAssignmentFile() {
|
||||
return assignmentFile;
|
||||
}
|
||||
|
||||
public void setAssignmentFile(String assignmentFile) {
|
||||
this.assignmentFile = assignmentFile;
|
||||
}
|
||||
|
||||
public String getDefaultFs() {
|
||||
return defaultFs;
|
||||
}
|
||||
|
||||
public void setDefaultFs(String defaultFs) {
|
||||
this.defaultFs = defaultFs;
|
||||
}
|
||||
|
||||
public List<String> getFsConfigResources() {
|
||||
return fsConfigResources;
|
||||
}
|
||||
|
||||
public void setFsConfigResources(List<String> fsConfigResources) {
|
||||
this.fsConfigResources = fsConfigResources;
|
||||
}
|
||||
|
||||
public void setControlBandPercentage(Double controlBandPercentage) {
|
||||
this.controlBandPercentage = controlBandPercentage;
|
||||
}
|
||||
|
||||
public Double getControlBandPercentage() {
|
||||
return controlBandPercentage;
|
||||
}
|
||||
|
||||
public void setCheckServerHealthInterval(long checkServerHealthInterval) {
|
||||
this.checkServerHealthInterval = checkServerHealthInterval;
|
||||
}
|
||||
|
||||
public long getCheckServerHealthInterval() {
|
||||
return checkServerHealthInterval;
|
||||
}
|
||||
|
||||
public int getServerFailuresBeforeDown() {
|
||||
return serverFailuresBeforeDown;
|
||||
}
|
||||
|
||||
public void setServerFailuresBeforeDown(int serverFailuresBeforeDown) {
|
||||
this.serverFailuresBeforeDown = serverFailuresBeforeDown;
|
||||
}
|
||||
|
||||
public int getServerSuccessesBeforeUp() {
|
||||
return serverSuccessesBeforeUp;
|
||||
}
|
||||
|
||||
public void setServerSuccessesBeforeUp(int serverSuccessesBeforeUp) {
|
||||
this.serverSuccessesBeforeUp = serverSuccessesBeforeUp;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package timely.balancer.configuration;
|
||||
|
||||
import javax.validation.Valid;
|
||||
|
||||
import org.apache.commons.pool2.impl.GenericKeyedObjectPoolConfig;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.boot.context.properties.NestedConfigurationProperty;
|
||||
import org.springframework.cloud.context.config.annotation.RefreshScope;
|
||||
|
||||
import timely.client.tcp.TcpClient;
|
||||
import timely.client.udp.UdpClient;
|
||||
import timely.common.configuration.ServerProperties;
|
||||
|
||||
@RefreshScope
|
||||
@ConfigurationProperties(prefix = "timely.balancer.server")
|
||||
public class BalancerServerProperties extends ServerProperties {
|
||||
|
||||
@Valid
|
||||
@NestedConfigurationProperty
|
||||
private GenericKeyedObjectPoolConfig<UdpClient> udpClientPool = new GenericKeyedObjectPoolConfig<>();
|
||||
|
||||
@Valid
|
||||
@NestedConfigurationProperty
|
||||
private GenericKeyedObjectPoolConfig<TcpClient> tcpClientPool = new GenericKeyedObjectPoolConfig<>();
|
||||
|
||||
private int numTcpPools = 1;
|
||||
|
||||
private int tcpBufferSize = -1;
|
||||
|
||||
public GenericKeyedObjectPoolConfig<UdpClient> getUdpClientPool() {
|
||||
return udpClientPool;
|
||||
}
|
||||
|
||||
public GenericKeyedObjectPoolConfig<TcpClient> getTcpClientPool() {
|
||||
return tcpClientPool;
|
||||
}
|
||||
|
||||
public int getNumTcpPools() {
|
||||
return numTcpPools;
|
||||
}
|
||||
|
||||
public void setNumTcpPools(int numTcpPools) {
|
||||
this.numTcpPools = numTcpPools;
|
||||
}
|
||||
|
||||
public int getTcpBufferSize() {
|
||||
return tcpBufferSize;
|
||||
}
|
||||
|
||||
public void setTcpBufferSize(int tcpBufferSize) {
|
||||
this.tcpBufferSize = tcpBufferSize;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package timely.balancer.configuration;
|
||||
|
||||
import javax.validation.Valid;
|
||||
|
||||
import org.apache.commons.pool2.impl.GenericKeyedObjectPoolConfig;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.boot.context.properties.NestedConfigurationProperty;
|
||||
import org.springframework.cloud.context.config.annotation.RefreshScope;
|
||||
|
||||
import timely.client.websocket.subscription.WebSocketSubscriptionClient;
|
||||
import timely.common.configuration.WebsocketProperties;
|
||||
|
||||
@RefreshScope
|
||||
@ConfigurationProperties(prefix = "timely.balancer.websocket")
|
||||
public class BalancerWebsocketProperties extends WebsocketProperties {
|
||||
|
||||
@Valid
|
||||
@NestedConfigurationProperty
|
||||
private GenericKeyedObjectPoolConfig<WebSocketSubscriptionClient> wsClientPool = new GenericKeyedObjectPoolConfig<>();
|
||||
|
||||
public GenericKeyedObjectPoolConfig<WebSocketSubscriptionClient> getWsClientPool() {
|
||||
return wsClientPool;
|
||||
}
|
||||
|
||||
private int incomingBufferSize = 50000000;
|
||||
|
||||
public int getIncomingBufferSize() {
|
||||
return incomingBufferSize;
|
||||
}
|
||||
|
||||
public void setIncomingBufferSize(int incomingBufferSize) {
|
||||
this.incomingBufferSize = incomingBufferSize;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
package timely.balancer.connection;
|
||||
|
||||
import org.apache.commons.lang3.builder.EqualsBuilder;
|
||||
import org.apache.commons.lang3.builder.HashCodeBuilder;
|
||||
import org.apache.commons.lang3.builder.ToStringBuilder;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import timely.balancer.ArrivalRate;
|
||||
import timely.balancer.configuration.BalancerProperties;
|
||||
|
||||
public class TimelyBalancedHost {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(TimelyBalancedHost.class);
|
||||
|
||||
private String host;
|
||||
private int tcpPort;
|
||||
private int httpPort;
|
||||
private int wsPort;
|
||||
private int udpPort;
|
||||
private boolean isUp = true;
|
||||
private BalancerProperties balancerProperties;
|
||||
private int failures = 0;
|
||||
private int successes = 0;
|
||||
private ArrivalRate arrivalRate = null;
|
||||
|
||||
private TimelyBalancedHost() {
|
||||
|
||||
}
|
||||
|
||||
private TimelyBalancedHost(String host, int tcpPort, int httpPort, int wsPort, int udpPort, ArrivalRate arrivalRate) {
|
||||
this.host = host;
|
||||
this.tcpPort = tcpPort;
|
||||
this.httpPort = httpPort;
|
||||
this.wsPort = wsPort;
|
||||
this.udpPort = udpPort;
|
||||
this.arrivalRate = arrivalRate;
|
||||
}
|
||||
|
||||
static public TimelyBalancedHost of(String host, int tcpPort, int httpPort, int wsPort, int udpPort, ArrivalRate arrivalRate) {
|
||||
return new TimelyBalancedHost(host, tcpPort, httpPort, wsPort, udpPort, arrivalRate);
|
||||
}
|
||||
|
||||
public void setBalancerProperties(BalancerProperties balancerProperties) {
|
||||
this.balancerProperties = balancerProperties;
|
||||
}
|
||||
|
||||
synchronized public boolean isUp() {
|
||||
return isUp;
|
||||
}
|
||||
|
||||
public void reportSuccess() {
|
||||
int serverSuccessesBeforeUp = balancerProperties.getServerSuccessesBeforeUp();
|
||||
String h = host;
|
||||
int p = tcpPort;
|
||||
synchronized (this) {
|
||||
if (log.isTraceEnabled()) {
|
||||
log.trace("success reported host:{} port:{} isUp:{} successes:{} serverSuccessesBeforeUp:{}", h, p, isUp, (successes + 1),
|
||||
serverSuccessesBeforeUp);
|
||||
}
|
||||
if (!isUp) {
|
||||
if (++successes >= serverSuccessesBeforeUp) {
|
||||
isUp = true;
|
||||
successes = 0;
|
||||
log.info("host up host:{} port:{}", h, p);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void reportFailure() {
|
||||
int serverFailuresBeforeDown = balancerProperties.getServerFailuresBeforeDown();
|
||||
String h = host;
|
||||
int p = tcpPort;
|
||||
synchronized (this) {
|
||||
if (log.isTraceEnabled()) {
|
||||
log.trace("failure reported host:{} port:{} isUp:{} failures:{} serverFailuresBeforeUp:{}", h, p, isUp, (failures + 1),
|
||||
serverFailuresBeforeDown);
|
||||
}
|
||||
if (isUp) {
|
||||
if (++failures >= serverFailuresBeforeDown) {
|
||||
isUp = false;
|
||||
failures = 0;
|
||||
log.info("host down host:{} port:{}", h, p);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
public void setHost(String host) {
|
||||
this.host = host;
|
||||
}
|
||||
|
||||
public int getUdpPort() {
|
||||
return udpPort;
|
||||
}
|
||||
|
||||
public void setUdpPort(int udpPort) {
|
||||
this.udpPort = udpPort;
|
||||
}
|
||||
|
||||
public int getTcpPort() {
|
||||
return tcpPort;
|
||||
}
|
||||
|
||||
public void setTcpPort(int tcpPort) {
|
||||
this.tcpPort = tcpPort;
|
||||
}
|
||||
|
||||
public int getHttpPort() {
|
||||
return httpPort;
|
||||
}
|
||||
|
||||
public void setHttpPort(int httpPort) {
|
||||
this.httpPort = httpPort;
|
||||
}
|
||||
|
||||
public int getWsPort() {
|
||||
return wsPort;
|
||||
}
|
||||
|
||||
public void setWsPort(int wsPort) {
|
||||
this.wsPort = wsPort;
|
||||
}
|
||||
|
||||
public void arrived() {
|
||||
if (this.arrivalRate != null) {
|
||||
this.arrivalRate.arrived();
|
||||
}
|
||||
}
|
||||
|
||||
public double getArrivalRate() {
|
||||
if (this.arrivalRate == null) {
|
||||
return 0;
|
||||
} else {
|
||||
return this.arrivalRate.getRate();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
HashCodeBuilder builder = new HashCodeBuilder();
|
||||
builder.append(host);
|
||||
builder.append(tcpPort);
|
||||
builder.append(httpPort);
|
||||
builder.append(wsPort);
|
||||
builder.append(udpPort);
|
||||
return builder.toHashCode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (o instanceof TimelyBalancedHost) {
|
||||
TimelyBalancedHost other = (TimelyBalancedHost) o;
|
||||
EqualsBuilder builder = new EqualsBuilder();
|
||||
builder.append(host, other.host);
|
||||
builder.append(tcpPort, other.tcpPort);
|
||||
builder.append(httpPort, other.httpPort);
|
||||
builder.append(wsPort, other.wsPort);
|
||||
builder.append(udpPort, other.udpPort);
|
||||
return builder.isEquals();
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
ToStringBuilder builder = new ToStringBuilder(this);
|
||||
builder.append(host);
|
||||
builder.append(tcpPort);
|
||||
builder.append(httpPort);
|
||||
builder.append(wsPort);
|
||||
builder.append(udpPort);
|
||||
builder.append(isUp);
|
||||
return builder.toString();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package timely.balancer.connection.http;
|
||||
|
||||
import javax.net.ssl.SSLContext;
|
||||
|
||||
import org.apache.commons.pool2.KeyedPooledObjectFactory;
|
||||
import org.apache.commons.pool2.PooledObject;
|
||||
import org.apache.commons.pool2.impl.DefaultPooledObject;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
|
||||
import timely.balancer.connection.TimelyBalancedHost;
|
||||
import timely.client.http.HttpClient;
|
||||
import timely.common.configuration.SslClientProperties;
|
||||
|
||||
public class HttpClientFactory implements KeyedPooledObjectFactory<TimelyBalancedHost,CloseableHttpClient> {
|
||||
|
||||
private final SSLContext sslContext;
|
||||
private SslClientProperties sslClientProperties;
|
||||
|
||||
public HttpClientFactory(SSLContext sslContext, SslClientProperties sslClientProperties) {
|
||||
this.sslContext = sslContext;
|
||||
this.sslClientProperties = sslClientProperties;
|
||||
}
|
||||
|
||||
@Override
|
||||
public PooledObject<CloseableHttpClient> makeObject(TimelyBalancedHost k) throws Exception {
|
||||
// disable cookie management because we are sharing connections
|
||||
return new DefaultPooledObject<>(
|
||||
HttpClient.get(this.sslContext, null, sslClientProperties.isHostVerificationEnabled(), sslClientProperties.isUseClientCert()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroyObject(TimelyBalancedHost k, PooledObject<CloseableHttpClient> o) throws Exception {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean validateObject(TimelyBalancedHost k, PooledObject<CloseableHttpClient> o) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void activateObject(TimelyBalancedHost k, PooledObject<CloseableHttpClient> o) throws Exception {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void passivateObject(TimelyBalancedHost k, PooledObject<CloseableHttpClient> o) throws Exception {
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package timely.balancer.connection.http;
|
||||
|
||||
import javax.net.ssl.SSLContext;
|
||||
|
||||
import org.apache.commons.pool2.impl.GenericKeyedObjectPool;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
|
||||
import timely.balancer.configuration.BalancerHttpProperties;
|
||||
import timely.balancer.connection.TimelyBalancedHost;
|
||||
import timely.common.configuration.SslClientProperties;
|
||||
|
||||
public class HttpClientPool extends GenericKeyedObjectPool<TimelyBalancedHost,CloseableHttpClient> {
|
||||
|
||||
public HttpClientPool(BalancerHttpProperties http, SSLContext sslContext, SslClientProperties sslClientProperties) {
|
||||
super(new HttpClientFactory(sslContext, sslClientProperties), http.getHttpClientPool());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package timely.balancer.connection.tcp;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.commons.pool2.KeyedPooledObjectFactory;
|
||||
import org.apache.commons.pool2.PooledObject;
|
||||
import org.apache.commons.pool2.impl.DefaultPooledObject;
|
||||
|
||||
import timely.balancer.connection.TimelyBalancedHost;
|
||||
import timely.client.tcp.TcpClient;
|
||||
|
||||
public class TcpClientFactory implements KeyedPooledObjectFactory<TimelyBalancedHost,TcpClient> {
|
||||
|
||||
private int bufferSize;
|
||||
|
||||
public TcpClientFactory(int bufferSize) {
|
||||
this.bufferSize = bufferSize;
|
||||
}
|
||||
|
||||
@Override
|
||||
public PooledObject<TcpClient> makeObject(TimelyBalancedHost k) throws Exception {
|
||||
return new DefaultPooledObject<>(new TcpClient(k.getHost(), k.getTcpPort(), bufferSize));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroyObject(TimelyBalancedHost k, PooledObject<TcpClient> o) throws Exception {
|
||||
try {
|
||||
o.getObject().flush();
|
||||
o.getObject().close();
|
||||
} catch (IOException e) {
|
||||
throw new IOException("Error closing connection to " + k.getHost() + ":" + k.getTcpPort());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean validateObject(TimelyBalancedHost k, PooledObject<TcpClient> o) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void activateObject(TimelyBalancedHost k, PooledObject<TcpClient> o) throws Exception {
|
||||
try {
|
||||
o.getObject().open();
|
||||
} catch (Exception e) {
|
||||
throw new IOException("Unable to connect to " + k.getHost() + ":" + k.getTcpPort(), e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void passivateObject(TimelyBalancedHost k, PooledObject<TcpClient> o) throws Exception {
|
||||
try {
|
||||
o.getObject().flush();
|
||||
} catch (Exception e) {
|
||||
throw new IOException("Error flushing connection to " + k.getHost() + ":" + k.getTcpPort());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package timely.balancer.connection.tcp;
|
||||
|
||||
import org.apache.commons.pool2.impl.GenericKeyedObjectPool;
|
||||
|
||||
import timely.balancer.configuration.BalancerServerProperties;
|
||||
import timely.balancer.connection.TimelyBalancedHost;
|
||||
import timely.client.tcp.TcpClient;
|
||||
|
||||
public class TcpClientPool extends GenericKeyedObjectPool<TimelyBalancedHost,TcpClient> {
|
||||
|
||||
public TcpClientPool(BalancerServerProperties balancerServerProperties) {
|
||||
super(new TcpClientFactory(balancerServerProperties.getTcpBufferSize()), balancerServerProperties.getTcpClientPool());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package timely.balancer.connection.udp;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.commons.pool2.KeyedPooledObjectFactory;
|
||||
import org.apache.commons.pool2.PooledObject;
|
||||
import org.apache.commons.pool2.impl.DefaultPooledObject;
|
||||
|
||||
import timely.balancer.connection.TimelyBalancedHost;
|
||||
import timely.client.udp.UdpClient;
|
||||
|
||||
public class UdpClientFactory implements KeyedPooledObjectFactory<TimelyBalancedHost,UdpClient> {
|
||||
|
||||
public UdpClientFactory() {}
|
||||
|
||||
@Override
|
||||
public PooledObject<UdpClient> makeObject(TimelyBalancedHost k) throws Exception {
|
||||
return new DefaultPooledObject<>(new UdpClient(k.getHost(), k.getUdpPort()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroyObject(TimelyBalancedHost k, PooledObject<UdpClient> o) throws Exception {
|
||||
try {
|
||||
o.getObject().flush();
|
||||
o.getObject().close();
|
||||
} catch (IOException e) {
|
||||
throw new IOException("Error closing connection to " + k.getHost() + ":" + k.getUdpPort());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean validateObject(TimelyBalancedHost k, PooledObject<UdpClient> o) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void activateObject(TimelyBalancedHost k, PooledObject<UdpClient> o) throws Exception {
|
||||
try {
|
||||
o.getObject().open();
|
||||
} catch (Exception e) {
|
||||
throw new IOException("Unable to connect to " + k.getHost() + ":" + k.getUdpPort(), e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void passivateObject(TimelyBalancedHost k, PooledObject<UdpClient> o) throws Exception {
|
||||
try {
|
||||
o.getObject().flush();
|
||||
} catch (Exception e) {
|
||||
throw new IOException("Error flushing connection to " + k.getHost() + ":" + k.getUdpPort());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package timely.balancer.connection.udp;
|
||||
|
||||
import org.apache.commons.pool2.impl.GenericKeyedObjectPool;
|
||||
|
||||
import timely.balancer.configuration.BalancerServerProperties;
|
||||
import timely.balancer.connection.TimelyBalancedHost;
|
||||
import timely.client.udp.UdpClient;
|
||||
|
||||
public class UdpClientPool extends GenericKeyedObjectPool<TimelyBalancedHost,UdpClient> {
|
||||
|
||||
public UdpClientPool(BalancerServerProperties serverProperties) {
|
||||
super(new UdpClientFactory(), serverProperties.getUdpClientPool());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package timely.balancer.connection.ws;
|
||||
|
||||
import javax.net.ssl.SSLContext;
|
||||
|
||||
import org.apache.commons.pool2.KeyedPooledObjectFactory;
|
||||
import org.apache.commons.pool2.PooledObject;
|
||||
import org.apache.commons.pool2.impl.DefaultPooledObject;
|
||||
|
||||
import timely.balancer.configuration.BalancerProperties;
|
||||
import timely.balancer.configuration.BalancerWebsocketProperties;
|
||||
import timely.balancer.connection.TimelyBalancedHost;
|
||||
import timely.client.websocket.subscription.WebSocketSubscriptionClient;
|
||||
import timely.common.configuration.SslClientProperties;
|
||||
|
||||
public class WsClientFactory implements KeyedPooledObjectFactory<TimelyBalancedHost,WebSocketSubscriptionClient> {
|
||||
|
||||
private final SSLContext sslContext;
|
||||
private BalancerProperties balancerProperties;
|
||||
private BalancerWebsocketProperties balancerWebsocketProperties;
|
||||
private SslClientProperties sslClientProperties;
|
||||
|
||||
public WsClientFactory(BalancerProperties balancerProperties, BalancerWebsocketProperties balancerWebsocketProperties,
|
||||
SslClientProperties sslClientProperties, SSLContext sslContext) {
|
||||
this.balancerProperties = balancerProperties;
|
||||
this.balancerWebsocketProperties = balancerWebsocketProperties;
|
||||
this.sslClientProperties = sslClientProperties;
|
||||
this.sslContext = sslContext;
|
||||
}
|
||||
|
||||
@Override
|
||||
public PooledObject<WebSocketSubscriptionClient> makeObject(TimelyBalancedHost k) throws Exception {
|
||||
|
||||
int bufferSize = balancerWebsocketProperties.getIncomingBufferSize();
|
||||
WebSocketSubscriptionClient client = new WebSocketSubscriptionClient(sslContext, k.getHost(), k.getHttpPort(), k.getWsPort(),
|
||||
sslClientProperties.isUseClientCert(), balancerProperties.isLoginRequired(), sslClientProperties.isHostVerificationEnabled(),
|
||||
bufferSize);
|
||||
return new DefaultPooledObject<>(client);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroyObject(TimelyBalancedHost k, PooledObject<WebSocketSubscriptionClient> o) throws Exception {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean validateObject(TimelyBalancedHost k, PooledObject<WebSocketSubscriptionClient> o) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void activateObject(TimelyBalancedHost k, PooledObject<WebSocketSubscriptionClient> o) throws Exception {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void passivateObject(TimelyBalancedHost k, PooledObject<WebSocketSubscriptionClient> o) throws Exception {
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package timely.balancer.connection.ws;
|
||||
|
||||
import javax.net.ssl.SSLContext;
|
||||
|
||||
import org.apache.commons.pool2.impl.GenericKeyedObjectPool;
|
||||
|
||||
import timely.balancer.configuration.BalancerProperties;
|
||||
import timely.balancer.configuration.BalancerWebsocketProperties;
|
||||
import timely.balancer.connection.TimelyBalancedHost;
|
||||
import timely.client.websocket.subscription.WebSocketSubscriptionClient;
|
||||
import timely.common.configuration.SslClientProperties;
|
||||
|
||||
public class WsClientPool extends GenericKeyedObjectPool<TimelyBalancedHost,WebSocketSubscriptionClient> {
|
||||
|
||||
public WsClientPool(BalancerProperties balancerProperties, BalancerWebsocketProperties balancerWebsocketProperties, SslClientProperties sslClientProperties,
|
||||
SSLContext sslContext) {
|
||||
super(new WsClientFactory(balancerProperties, balancerWebsocketProperties, sslClientProperties, sslContext),
|
||||
balancerWebsocketProperties.getWsClientPool());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
package timely.balancer.healthcheck;
|
||||
|
||||
import timely.balancer.connection.TimelyBalancedHost;
|
||||
|
||||
public interface ServerHealthCheck {
|
||||
|
||||
public boolean isServerHealthy(TimelyBalancedHost timelyBalancedHost);
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package timely.balancer.healthcheck;
|
||||
|
||||
import timely.balancer.connection.TimelyBalancedHost;
|
||||
import timely.balancer.connection.tcp.TcpClientPool;
|
||||
import timely.client.tcp.TcpClient;
|
||||
|
||||
public class TCPServerHealthCheck implements ServerHealthCheck {
|
||||
|
||||
final private TcpClientPool tcpClientPool;
|
||||
|
||||
public TCPServerHealthCheck(TcpClientPool tcpClientPool) {
|
||||
this.tcpClientPool = tcpClientPool;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isServerHealthy(TimelyBalancedHost timelyBalancedHost) {
|
||||
|
||||
boolean healthy = true;
|
||||
TcpClient client = null;
|
||||
try {
|
||||
client = tcpClientPool.borrowObject(timelyBalancedHost);
|
||||
client.open();
|
||||
} catch (Exception e) {
|
||||
healthy = false;
|
||||
|
||||
} finally {
|
||||
if (client != null) {
|
||||
tcpClientPool.returnObject(timelyBalancedHost, client);
|
||||
}
|
||||
}
|
||||
return healthy;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
package timely.balancer.netty.http;
|
||||
|
||||
import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_LENGTH;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.net.URLEncoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.http.Header;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.client.methods.HttpPost;
|
||||
import org.apache.http.client.methods.HttpUriRequest;
|
||||
import org.apache.http.entity.StringEntity;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.apache.http.message.BasicHeader;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.util.StreamUtils;
|
||||
|
||||
import com.google.common.collect.Multimap;
|
||||
|
||||
import io.netty.buffer.Unpooled;
|
||||
import io.netty.channel.ChannelHandlerContext;
|
||||
import io.netty.channel.SimpleChannelInboundHandler;
|
||||
import io.netty.handler.codec.http.DefaultFullHttpResponse;
|
||||
import io.netty.handler.codec.http.FullHttpRequest;
|
||||
import io.netty.handler.codec.http.FullHttpResponse;
|
||||
import io.netty.handler.codec.http.HttpMethod;
|
||||
import io.netty.handler.codec.http.HttpResponseStatus;
|
||||
import io.netty.handler.codec.http.HttpVersion;
|
||||
import io.netty.util.ReferenceCountUtil;
|
||||
import timely.api.request.AuthenticatedRequest;
|
||||
import timely.api.request.HttpRequest;
|
||||
import timely.api.request.MetricRequest;
|
||||
import timely.api.request.timeseries.QueryRequest;
|
||||
import timely.api.response.TimelyException;
|
||||
import timely.auth.TimelyAuthenticationToken;
|
||||
import timely.auth.util.HttpHeaderUtils;
|
||||
import timely.auth.util.ProxiedEntityUtils;
|
||||
import timely.balancer.MetricResolver;
|
||||
import timely.balancer.connection.TimelyBalancedHost;
|
||||
import timely.balancer.connection.http.HttpClientPool;
|
||||
import timely.netty.http.TimelyHttpHandler;
|
||||
|
||||
public class HttpRelayHandler extends SimpleChannelInboundHandler<HttpRequest> implements TimelyHttpHandler {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(HttpRelayHandler.class);
|
||||
private final HttpClientPool httpClientPool;
|
||||
private MetricResolver metricResolver;
|
||||
|
||||
public HttpRelayHandler(MetricResolver metricResolver, HttpClientPool httpClientPool) {
|
||||
this.metricResolver = metricResolver;
|
||||
this.httpClientPool = httpClientPool;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void channelRead0(ChannelHandlerContext ctx, HttpRequest msg) throws Exception {
|
||||
|
||||
CloseableHttpClient client = null;
|
||||
TimelyBalancedHost k = null;
|
||||
CloseableHttpResponse relayedResponse = null;
|
||||
FullHttpRequest request = null;
|
||||
try {
|
||||
try {
|
||||
request = msg.getHttpRequest();
|
||||
String originalURI = request.uri();
|
||||
originalURI = encodeURI(originalURI);
|
||||
|
||||
String metric = null;
|
||||
if (msg instanceof QueryRequest) {
|
||||
Collection<QueryRequest.SubQuery> subqueries = ((QueryRequest) msg).getQueries();
|
||||
if (subqueries != null) {
|
||||
Iterator<QueryRequest.SubQuery> itr = subqueries.iterator();
|
||||
if (itr.hasNext()) {
|
||||
metric = itr.next().getMetric();
|
||||
}
|
||||
}
|
||||
k = metricResolver.getHostPortKey(metric);
|
||||
} else if (msg instanceof MetricRequest) {
|
||||
metric = ((MetricRequest) msg).getMetric().getName();
|
||||
k = metricResolver.getHostPortKeyIngest(metric);
|
||||
} else {
|
||||
k = metricResolver.getHostPortKey(null);
|
||||
}
|
||||
client = httpClientPool.borrowObject(k);
|
||||
|
||||
Multimap<String,String> headers = HttpHeaderUtils.toMultimap(request.headers());
|
||||
if (msg instanceof AuthenticatedRequest) {
|
||||
AuthenticatedRequest authenticatedRequest = (AuthenticatedRequest) msg;
|
||||
TimelyAuthenticationToken token = authenticatedRequest.getToken();
|
||||
if (token != null) {
|
||||
if (token.getClientCert() != null) {
|
||||
ProxiedEntityUtils.addProxyHeaders(headers, token.getClientCert());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
List<Header> relayedHeaderList = new ArrayList<>();
|
||||
for (Map.Entry<String,String> h : headers.entries()) {
|
||||
if (!h.getKey().equals(CONTENT_LENGTH.toString())) {
|
||||
relayedHeaderList.add(new BasicHeader(h.getKey(), h.getValue()));
|
||||
}
|
||||
}
|
||||
Header[] headerArray = new Header[relayedHeaderList.size()];
|
||||
relayedHeaderList.toArray(headerArray);
|
||||
|
||||
String relayURI = "https://" + k.getHost() + ":" + k.getHttpPort() + originalURI;
|
||||
if (request.method().equals(HttpMethod.GET)) {
|
||||
HttpUriRequest relayedRequest = new HttpGet(relayURI);
|
||||
if (headerArray.length > 0) {
|
||||
relayedRequest.setHeaders(headerArray);
|
||||
}
|
||||
relayedResponse = client.execute(relayedRequest);
|
||||
|
||||
} else if (request.method().equals(HttpMethod.POST)) {
|
||||
HttpPost relayedRequest = new HttpPost(relayURI);
|
||||
|
||||
String content = request.content().toString(StandardCharsets.UTF_8);
|
||||
StringEntity entity = new StringEntity(content);
|
||||
relayedRequest.setEntity(entity);
|
||||
if (headerArray.length > 0) {
|
||||
relayedRequest.setHeaders(headerArray);
|
||||
}
|
||||
relayedResponse = client.execute(relayedRequest);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
String message = e.getMessage();
|
||||
if (message == null) {
|
||||
log.error("", e);
|
||||
} else {
|
||||
if (message.contains("No matching tags")) {
|
||||
log.trace(message);
|
||||
} else {
|
||||
log.error(message, e);
|
||||
}
|
||||
}
|
||||
this.sendHttpError(ctx, new TimelyException(HttpResponseStatus.INTERNAL_SERVER_ERROR.code(), e.getMessage(), e.getLocalizedMessage(), e));
|
||||
return;
|
||||
} finally {
|
||||
if (request != null) {
|
||||
ReferenceCountUtil.release(request);
|
||||
}
|
||||
}
|
||||
FullHttpResponse response = null;
|
||||
if (relayedResponse == null) {
|
||||
this.sendHttpError(ctx, new TimelyException(HttpResponseStatus.METHOD_NOT_ALLOWED.code(), "Method not allowed", "Method not allowed"));
|
||||
} else {
|
||||
ByteArrayOutputStream baos = new MyByteArrayOutputStream();
|
||||
StreamUtils.copy(relayedResponse.getEntity().getContent(), baos);
|
||||
|
||||
response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.valueOf(relayedResponse.getStatusLine().getStatusCode()),
|
||||
Unpooled.copiedBuffer(baos.toByteArray()));
|
||||
for (Header h : relayedResponse.getAllHeaders()) {
|
||||
response.headers().add(h.getName(), h.getValue());
|
||||
}
|
||||
response.headers().set(CONTENT_LENGTH, response.content().readableBytes());
|
||||
sendResponse(ctx, response);
|
||||
}
|
||||
} finally {
|
||||
if (relayedResponse != null) {
|
||||
relayedResponse.close();
|
||||
}
|
||||
if (client != null && k != null) {
|
||||
httpClientPool.returnObject(k, client);
|
||||
} else {
|
||||
log.error("NOT RETURNING CONNECTION! " + msg.getHttpRequest().uri());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static public class MyByteArrayOutputStream extends ByteArrayOutputStream {
|
||||
|
||||
public MyByteArrayOutputStream() {}
|
||||
|
||||
public MyByteArrayOutputStream(int size) {
|
||||
super(size);
|
||||
}
|
||||
|
||||
public int getCount() {
|
||||
return count;
|
||||
}
|
||||
|
||||
public byte[] getBuf() {
|
||||
return buf;
|
||||
}
|
||||
}
|
||||
|
||||
static public String encodeURI(String s) {
|
||||
try {
|
||||
s = s.replaceAll("\\{", URLEncoder.encode("{", "UTF-8"));
|
||||
s = s.replaceAll("\\}", URLEncoder.encode("}", "UTF-8"));
|
||||
} catch (Exception e) {
|
||||
|
||||
}
|
||||
return s;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
package timely.balancer.netty.tcp;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import io.netty.buffer.Unpooled;
|
||||
import io.netty.channel.ChannelFuture;
|
||||
import io.netty.channel.ChannelHandlerContext;
|
||||
import io.netty.channel.SimpleChannelInboundHandler;
|
||||
import timely.api.request.MetricRequest;
|
||||
import timely.api.request.TcpRequest;
|
||||
import timely.balancer.MetricResolver;
|
||||
import timely.balancer.connection.TimelyBalancedHost;
|
||||
import timely.balancer.connection.tcp.TcpClientPool;
|
||||
import timely.client.tcp.TcpClient;
|
||||
import timely.netty.Constants;
|
||||
|
||||
public class TcpRelayHandler extends SimpleChannelInboundHandler<TcpRequest> {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(TcpRelayHandler.class);
|
||||
private static final String LOG_ERR_MSG = "Error storing put metric: {}";
|
||||
private static final String ERR_MSG = "Error storing put metric: ";
|
||||
|
||||
private MetricResolver metricResolver;
|
||||
private List<TcpClientPool> tcpClientPools;
|
||||
|
||||
public TcpRelayHandler(MetricResolver metricResolver, List<TcpClientPool> tcpClientPools) {
|
||||
this.metricResolver = metricResolver;
|
||||
this.tcpClientPools = tcpClientPools;
|
||||
}
|
||||
|
||||
private TcpClientPool tcpClientPool() {
|
||||
int x = Math.toIntExact(Thread.currentThread().getId() % tcpClientPools.size());
|
||||
return tcpClientPools.get(x);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void channelRead0(ChannelHandlerContext ctx, TcpRequest msg) throws Exception {
|
||||
log.trace("Received {}", msg);
|
||||
try {
|
||||
String line;
|
||||
Pair<TimelyBalancedHost,TcpClient> keyClientPair = null;
|
||||
try {
|
||||
if (msg instanceof MetricRequest) {
|
||||
String metricName = ((MetricRequest) msg).getMetric().getName();
|
||||
line = ((MetricRequest) msg).getLine();
|
||||
keyClientPair = getClient(metricName, true);
|
||||
} else {
|
||||
// Version request
|
||||
line = "version";
|
||||
keyClientPair = getClient(null, false);
|
||||
}
|
||||
keyClientPair.getRight().write(line + "\n");
|
||||
} finally {
|
||||
if (keyClientPair != null && keyClientPair.getLeft() != null && keyClientPair.getRight() != null) {
|
||||
tcpClientPool().returnObject(keyClientPair.getLeft(), keyClientPair.getRight());
|
||||
}
|
||||
}
|
||||
|
||||
} catch (IOException e) {
|
||||
log.error(LOG_ERR_MSG, msg, e);
|
||||
ChannelFuture cf = ctx.writeAndFlush(Unpooled.copiedBuffer((ERR_MSG + e.getMessage() + "\n").getBytes(StandardCharsets.UTF_8)));
|
||||
if (!cf.isSuccess()) {
|
||||
log.error(Constants.ERR_WRITING_RESPONSE, cf.cause());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Pair<TimelyBalancedHost,TcpClient> getClient(String metric, boolean metricRequest) {
|
||||
TcpClient client = null;
|
||||
TimelyBalancedHost k = null;
|
||||
int failures = 0;
|
||||
while (client == null) {
|
||||
try {
|
||||
k = (metricRequest == true) ? metricResolver.getHostPortKeyIngest(metric) : metricResolver.getHostPortKey(metric);
|
||||
client = tcpClientPool().borrowObject(k);
|
||||
} catch (Exception e1) {
|
||||
failures++;
|
||||
client = null;
|
||||
if (failures % 10 == 0) {
|
||||
log.error(e1.getMessage(), e1);
|
||||
}
|
||||
try {
|
||||
Thread.sleep(failures < 10 ? 500 : 60000);
|
||||
} catch (InterruptedException e2) {
|
||||
// nothing
|
||||
}
|
||||
}
|
||||
}
|
||||
return Pair.of(k, client);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package timely.balancer.netty.udp;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import io.netty.buffer.Unpooled;
|
||||
import io.netty.channel.ChannelFuture;
|
||||
import io.netty.channel.ChannelHandlerContext;
|
||||
import io.netty.channel.SimpleChannelInboundHandler;
|
||||
import timely.api.request.MetricRequest;
|
||||
import timely.api.request.UdpRequest;
|
||||
import timely.balancer.MetricResolver;
|
||||
import timely.balancer.connection.TimelyBalancedHost;
|
||||
import timely.balancer.connection.udp.UdpClientPool;
|
||||
import timely.client.udp.UdpClient;
|
||||
import timely.netty.Constants;
|
||||
|
||||
public class UdpRelayHandler extends SimpleChannelInboundHandler<UdpRequest> {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(UdpRelayHandler.class);
|
||||
private static final String LOG_ERR_MSG = "Error storing put metric: {}";
|
||||
private static final String ERR_MSG = "Error storing put metric: ";
|
||||
|
||||
private MetricResolver metricResolver;
|
||||
private UdpClientPool udpClientPool;
|
||||
|
||||
public UdpRelayHandler(MetricResolver metricResolver, UdpClientPool udpClientPool) {
|
||||
this.metricResolver = metricResolver;
|
||||
this.udpClientPool = udpClientPool;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void channelRead0(ChannelHandlerContext ctx, UdpRequest msg) throws Exception {
|
||||
log.trace("Received {}", msg);
|
||||
try {
|
||||
MetricRequest metricRequest = (MetricRequest) msg;
|
||||
String metricName = metricRequest.getMetric().getName();
|
||||
Pair<TimelyBalancedHost,UdpClient> keyClientPair = null;
|
||||
try {
|
||||
keyClientPair = getClient(metricName, true);
|
||||
keyClientPair.getRight().write(metricRequest.getLine() + "\n");
|
||||
keyClientPair.getRight().flush();
|
||||
} finally {
|
||||
if (keyClientPair != null && keyClientPair.getLeft() != null && keyClientPair.getRight() != null) {
|
||||
udpClientPool.returnObject(keyClientPair.getLeft(), keyClientPair.getRight());
|
||||
}
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error(LOG_ERR_MSG, msg, e);
|
||||
ChannelFuture cf = ctx.writeAndFlush(Unpooled.copiedBuffer((ERR_MSG + e.getMessage() + "\n").getBytes(StandardCharsets.UTF_8)));
|
||||
if (!cf.isSuccess()) {
|
||||
log.error(Constants.ERR_WRITING_RESPONSE, cf.cause());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Pair<TimelyBalancedHost,UdpClient> getClient(String metric, boolean metricRequest) {
|
||||
UdpClient client = null;
|
||||
TimelyBalancedHost k = null;
|
||||
int failures = 0;
|
||||
while (client == null) {
|
||||
try {
|
||||
k = (metricRequest == true) ? metricResolver.getHostPortKeyIngest(metric) : metricResolver.getHostPortKey(metric);
|
||||
client = udpClientPool.borrowObject(k);
|
||||
} catch (Exception e1) {
|
||||
failures++;
|
||||
client = null;
|
||||
log.error(e1.getMessage(), e1);
|
||||
try {
|
||||
Thread.sleep(failures < 10 ? 500 : 60000);
|
||||
} catch (InterruptedException e2) {
|
||||
// nothing
|
||||
}
|
||||
}
|
||||
}
|
||||
return Pair.of(k, client);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package timely.balancer.netty.ws;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import javax.websocket.CloseReason;
|
||||
import javax.websocket.EndpointConfig;
|
||||
import javax.websocket.Session;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.google.common.collect.HashMultimap;
|
||||
import com.google.common.collect.Multimap;
|
||||
|
||||
import io.netty.channel.ChannelHandlerContext;
|
||||
import io.netty.handler.codec.http.HttpResponseStatus;
|
||||
import io.netty.handler.codec.http.websocketx.PingWebSocketFrame;
|
||||
import io.netty.handler.codec.http.websocketx.TextWebSocketFrame;
|
||||
import io.netty.util.concurrent.ScheduledFuture;
|
||||
import timely.api.response.TimelyException;
|
||||
import timely.auth.TimelyAuthenticationToken;
|
||||
import timely.auth.util.ProxiedEntityUtils;
|
||||
import timely.client.websocket.ClientHandler;
|
||||
|
||||
public class WsClientHandler extends ClientHandler {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(WsClientHandler.class);
|
||||
private final ChannelHandlerContext ctx;
|
||||
private final TimelyAuthenticationToken token;
|
||||
private ScheduledFuture<?> ping;
|
||||
|
||||
public WsClientHandler(ChannelHandlerContext ctx, TimelyAuthenticationToken token, int pingRate) {
|
||||
this.ctx = ctx;
|
||||
this.token = token;
|
||||
this.ping = this.ctx.executor().scheduleAtFixedRate(() -> {
|
||||
log.trace("Sending ping on channel {}", ctx.channel());
|
||||
ctx.writeAndFlush(new PingWebSocketFrame());
|
||||
}, pingRate, pingRate, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void beforeRequest(Map<String,List<String>> headers) {
|
||||
if (token.getClientCert() != null) {
|
||||
Multimap<String,String> proxyRequestHeaders = HashMultimap.create();
|
||||
ProxiedEntityUtils.addProxyHeaders(proxyRequestHeaders, token.getClientCert());
|
||||
for (String s : proxyRequestHeaders.keySet()) {
|
||||
headers.put(s, new ArrayList<>(proxyRequestHeaders.get(s)));
|
||||
}
|
||||
}
|
||||
|
||||
Multimap<String,String> originalRequestHeaders = token.getHttpHeaders();
|
||||
for (String s : originalRequestHeaders.keySet()) {
|
||||
if (!headers.containsKey(s)) {
|
||||
// add pre-existing values if key does not exist in proxyRequest
|
||||
headers.put(s, new ArrayList<>(originalRequestHeaders.get(s)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onOpen(Session session, EndpointConfig config) {
|
||||
session.addMessageHandler(String.class, message -> {
|
||||
ctx.writeAndFlush(new TextWebSocketFrame(message));
|
||||
log.debug("Message received on Websocket session {}: {}", session.getId(), message);
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onClose(Session session, CloseReason reason) {
|
||||
super.onClose(session, reason);
|
||||
if (!reason.getCloseCode().equals(CloseReason.CloseCodes.NORMAL_CLOSURE)) {
|
||||
log.error("Abnormal close: " + reason.getReasonPhrase());
|
||||
Exception e = new TimelyException(HttpResponseStatus.INTERNAL_SERVER_ERROR.code(), reason.getReasonPhrase(), "");
|
||||
WsRelayHandler.sendErrorResponse(ctx, HttpResponseStatus.INTERNAL_SERVER_ERROR, e);
|
||||
}
|
||||
if (!ping.isCancelled()) {
|
||||
ping.cancel(false);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onError(Session session, Throwable t) {
|
||||
super.onError(session, t);
|
||||
log.error(t.getMessage(), t);
|
||||
Exception e = new TimelyException(HttpResponseStatus.INTERNAL_SERVER_ERROR.code(), t.getMessage(), "");
|
||||
WsRelayHandler.sendErrorResponse(ctx, HttpResponseStatus.INTERNAL_SERVER_ERROR, e);
|
||||
if (!ping.isCancelled()) {
|
||||
ping.cancel(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package timely.balancer.netty.ws;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import timely.balancer.connection.TimelyBalancedHost;
|
||||
import timely.balancer.connection.ws.WsClientPool;
|
||||
import timely.client.websocket.subscription.WebSocketSubscriptionClient;
|
||||
|
||||
public class WsClientHolder {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(WsClientHolder.class);
|
||||
private WebSocketSubscriptionClient client;
|
||||
private TimelyBalancedHost host;
|
||||
|
||||
public WsClientHolder(TimelyBalancedHost host, WebSocketSubscriptionClient client) {
|
||||
this.host = host;
|
||||
this.client = client;
|
||||
}
|
||||
|
||||
synchronized public WebSocketSubscriptionClient getClient() {
|
||||
return client;
|
||||
}
|
||||
|
||||
synchronized public TimelyBalancedHost getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
synchronized public void close(WsClientPool wsClientPool) {
|
||||
if (client != null && host != null) {
|
||||
try {
|
||||
client.close();
|
||||
} catch (IOException e) {
|
||||
log.error("Error closing web socket client", e);
|
||||
}
|
||||
wsClientPool.returnObject(host, client);
|
||||
client = null;
|
||||
host = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
package timely.balancer.netty.ws;
|
||||
|
||||
import java.net.ConnectException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.SerializationFeature;
|
||||
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
|
||||
import com.google.common.collect.Multimap;
|
||||
|
||||
import io.netty.channel.ChannelFuture;
|
||||
import io.netty.channel.ChannelFutureListener;
|
||||
import io.netty.channel.ChannelHandlerContext;
|
||||
import io.netty.channel.SimpleChannelInboundHandler;
|
||||
import io.netty.handler.codec.http.HttpResponseStatus;
|
||||
import io.netty.handler.codec.http.websocketx.CloseWebSocketFrame;
|
||||
import io.netty.handler.codec.http.websocketx.TextWebSocketFrame;
|
||||
import timely.api.request.websocket.AddSubscription;
|
||||
import timely.api.request.websocket.CloseSubscription;
|
||||
import timely.api.request.websocket.CreateSubscription;
|
||||
import timely.api.request.websocket.RemoveSubscription;
|
||||
import timely.api.request.websocket.SubscriptionRequest;
|
||||
import timely.api.response.TimelyException;
|
||||
import timely.auth.util.ProxiedEntityUtils;
|
||||
import timely.balancer.MetricResolver;
|
||||
import timely.balancer.configuration.BalancerWebsocketProperties;
|
||||
import timely.balancer.connection.TimelyBalancedHost;
|
||||
import timely.balancer.connection.ws.WsClientPool;
|
||||
import timely.client.websocket.subscription.WebSocketSubscriptionClient;
|
||||
import timely.netty.http.TimelyHttpHandler;
|
||||
|
||||
public class WsRelayHandler extends SimpleChannelInboundHandler<SubscriptionRequest> implements TimelyHttpHandler {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(WsRelayHandler.class);
|
||||
|
||||
private BalancerWebsocketProperties balancerWebsocketProperties;
|
||||
private WsClientPool wsClientPool;
|
||||
private MetricResolver metricResolver;
|
||||
static private ObjectMapper mapper;
|
||||
Map<String,Map<String,WsClientHolder>> wsClients = new ConcurrentHashMap<>();
|
||||
Map<String,ChannelHandlerContext> wsSubscriptions = new ConcurrentHashMap<>();
|
||||
|
||||
static {
|
||||
mapper = new ObjectMapper();
|
||||
mapper.registerModule(new Jdk8Module());
|
||||
mapper.configure(SerializationFeature.INDENT_OUTPUT, true);
|
||||
mapper.configure(SerializationFeature.WRAP_EXCEPTIONS, true);
|
||||
mapper.configure(SerializationFeature.WRAP_ROOT_VALUE, true);
|
||||
}
|
||||
|
||||
public WsRelayHandler(BalancerWebsocketProperties balancerWebsocketProperties, MetricResolver metricResolver, WsClientPool wsClientPool) {
|
||||
this.balancerWebsocketProperties = balancerWebsocketProperties;
|
||||
this.metricResolver = metricResolver;
|
||||
this.wsClientPool = wsClientPool;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void channelRead0(ChannelHandlerContext ctx, SubscriptionRequest msg) throws Exception {
|
||||
TimelyBalancedHost k;
|
||||
ChannelHandlerContext origContext = null;
|
||||
String subscriptionId;
|
||||
try {
|
||||
String metric = null;
|
||||
|
||||
Multimap<String,String> headers = msg.getRequestHeaders();
|
||||
ProxiedEntityUtils.addProxyHeaders(headers, msg.getToken().getClientCert());
|
||||
|
||||
if (msg instanceof CreateSubscription) {
|
||||
CreateSubscription create = (CreateSubscription) msg;
|
||||
final String currentSubscriptionId = create.getSubscriptionId();
|
||||
subscriptionId = currentSubscriptionId;
|
||||
wsSubscriptions.put(subscriptionId, ctx);
|
||||
// WebSocketSubscriptionClient sends the createSubscription when
|
||||
// we call open
|
||||
// We don't know where to send it to yet anyway because that
|
||||
// depends on the metric.
|
||||
ctx.channel().closeFuture().addListener(new ChannelFutureListener() {
|
||||
|
||||
@Override
|
||||
public void operationComplete(ChannelFuture future) throws Exception {
|
||||
Map<String,WsClientHolder> metricToClientMap = wsClients.get(currentSubscriptionId);
|
||||
|
||||
if (metricToClientMap != null) {
|
||||
synchronized (metricToClientMap) {
|
||||
// close all clients for this subscriptionId
|
||||
log.debug("Channel closed, closing subscriptions for subscriptionId:{}", currentSubscriptionId);
|
||||
for (Map.Entry<String,WsClientHolder> entry : metricToClientMap.entrySet()) {
|
||||
entry.getValue().close(wsClientPool);
|
||||
}
|
||||
wsClients.remove(currentSubscriptionId);
|
||||
wsSubscriptions.remove(currentSubscriptionId);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
} else if (msg instanceof AddSubscription) {
|
||||
AddSubscription add = (AddSubscription) msg;
|
||||
metric = add.getMetric();
|
||||
subscriptionId = add.getSubscriptionId();
|
||||
origContext = wsSubscriptions.get(subscriptionId);
|
||||
if (origContext == null) {
|
||||
// send error because user never sent a CreateSubscription
|
||||
// and we have nowhere to send the results
|
||||
log.info("ChannelHandlerContext not found for subscriptionId:{} - createSubscription not called?", subscriptionId);
|
||||
sendErrorResponse(ctx, HttpResponseStatus.BAD_REQUEST, new IllegalArgumentException("Must call create first"));
|
||||
}
|
||||
Map<String,WsClientHolder> metricToClientMap = wsClients.get(subscriptionId);
|
||||
if (metricToClientMap == null) {
|
||||
metricToClientMap = new HashMap<>();
|
||||
wsClients.put(subscriptionId, metricToClientMap);
|
||||
}
|
||||
WebSocketSubscriptionClient client = null;
|
||||
synchronized (metricToClientMap) {
|
||||
WsClientHolder hostClientHolder = metricToClientMap.get(metric);
|
||||
if (hostClientHolder == null) {
|
||||
k = metricResolver.getHostPortKey(metric);
|
||||
client = wsClientPool.borrowObject(k);
|
||||
client.open(new WsClientHandler(origContext, add.getToken(), (balancerWebsocketProperties.getTimeout() / 2)));
|
||||
metricToClientMap.put(metric, new WsClientHolder(k, client));
|
||||
} else {
|
||||
client = hostClientHolder.getClient();
|
||||
}
|
||||
}
|
||||
Map<String,String> tags = null;
|
||||
|
||||
Long startTime = 0L;
|
||||
Long endTime = 0L;
|
||||
Long delayTime = 5000L;
|
||||
if (add.getTags().isPresent()) {
|
||||
tags = add.getTags().get();
|
||||
}
|
||||
if (add.getStartTime().isPresent()) {
|
||||
startTime = add.getStartTime().get();
|
||||
}
|
||||
if (add.getEndTime().isPresent()) {
|
||||
endTime = add.getEndTime().get();
|
||||
}
|
||||
if (add.getDelayTime().isPresent()) {
|
||||
delayTime = add.getDelayTime().get();
|
||||
}
|
||||
|
||||
client.addSubscription(add.getMetric(), tags, startTime, endTime, delayTime);
|
||||
|
||||
} else if (msg instanceof RemoveSubscription) {
|
||||
RemoveSubscription remove = (RemoveSubscription) msg;
|
||||
metric = remove.getMetric();
|
||||
subscriptionId = remove.getSubscriptionId();
|
||||
Map<String,WsClientHolder> metricToClientMap = wsClients.get(subscriptionId);
|
||||
if (metricToClientMap != null) {
|
||||
synchronized (metricToClientMap) {
|
||||
WsClientHolder hostClientHolder = metricToClientMap.get(metric);
|
||||
if (hostClientHolder != null) {
|
||||
hostClientHolder.getClient().removeSubscription(metric);
|
||||
} else {
|
||||
log.info("client not found for subscriptionId:{} metric:{}", subscriptionId, metric);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (msg instanceof CloseSubscription) {
|
||||
CloseSubscription close = (CloseSubscription) msg;
|
||||
subscriptionId = close.getSubscriptionId();
|
||||
Map<String,WsClientHolder> metricToClientMap = wsClients.get(subscriptionId);
|
||||
if (metricToClientMap != null) {
|
||||
synchronized (metricToClientMap) {
|
||||
// close all clients for this subscriptionId
|
||||
for (Map.Entry<String,WsClientHolder> entry : metricToClientMap.entrySet()) {
|
||||
entry.getValue().close(wsClientPool);
|
||||
}
|
||||
}
|
||||
wsClients.remove(subscriptionId);
|
||||
wsSubscriptions.remove(subscriptionId);
|
||||
}
|
||||
ctx.writeAndFlush(new CloseWebSocketFrame(1000, "Client requested close."));
|
||||
}
|
||||
} catch (ConnectException e1) {
|
||||
log.error(e1.getMessage(), e1);
|
||||
ChannelHandlerContext errorCtx = origContext == null ? ctx : origContext;
|
||||
sendErrorResponse(errorCtx, HttpResponseStatus.INTERNAL_SERVER_ERROR, e1);
|
||||
} catch (Exception e2) {
|
||||
log.error(e2.getMessage(), e2);
|
||||
ChannelHandlerContext errorCtx = origContext == null ? ctx : origContext;
|
||||
sendErrorResponse(errorCtx, HttpResponseStatus.INTERNAL_SERVER_ERROR, e2);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
static public void sendErrorResponse(ChannelHandlerContext ctx, HttpResponseStatus status, Exception ex) {
|
||||
|
||||
try {
|
||||
TimelyException te = new TimelyException(status.code(), ex.getMessage(), ex.getMessage());
|
||||
String json = mapper.writeValueAsString(te);
|
||||
ctx.writeAndFlush(new TextWebSocketFrame(json));
|
||||
} catch (JsonProcessingException e) {
|
||||
log.error("Error serializing exception");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
===============================================================================
|
||||
_______ _ _ ____ _
|
||||
|__ __(_) | | | _ \ | |
|
||||
| | _ _ __ ___ ___| |_ _ | |_) | __ _| | __ _ _ __ ___ ___ _ __
|
||||
| | | | '_ ` _ \ / _ \ | | | | | _ < / _` | |/ _` | '_ \ / __/ _ \ '__|
|
||||
| | | | | | | | | __/ | |_| | | |_) | (_| | | (_| | | | | (_| __/ |
|
||||
|_| |_|_| |_| |_|\___|_|\__, | |____/ \__,_|_|\__,_|_| |_|\___\___|_|
|
||||
__/ |
|
||||
|___/
|
||||
===============================================================================
|
||||
TimelyBalancer ${application.formatted-version}
|
||||
@@ -0,0 +1,6 @@
|
||||
spring:
|
||||
application:
|
||||
name: timely-balancer
|
||||
config:
|
||||
name: timely-balancer
|
||||
location: ${CONF_DIR}/
|
||||
Executable
+2
@@ -0,0 +1,2 @@
|
||||
cd src/main/generated-java
|
||||
flatc --java ../resources/fb/metric.fbs
|
||||
@@ -0,0 +1,21 @@
|
||||
namespace timely.api.flatbuffer;
|
||||
|
||||
file_identifier "TMFB";
|
||||
|
||||
table Tag {
|
||||
key:string;
|
||||
value:string;
|
||||
}
|
||||
|
||||
table Metric {
|
||||
name:string;
|
||||
timestamp:long;
|
||||
value:double;
|
||||
tags:[Tag];
|
||||
}
|
||||
|
||||
table Metrics {
|
||||
metrics:[Metric];
|
||||
}
|
||||
|
||||
root_type Metrics;
|
||||
@@ -0,0 +1,21 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<FindBugsFilter>
|
||||
<Match>
|
||||
<Class name="timely.balancer.Balancer" />
|
||||
<Method name="shutdown" />
|
||||
<Bug pattern="UC_USELESS_OBJECT" />
|
||||
</Match>
|
||||
<Match>
|
||||
<Class name="timely.balancer.netty.http.HttpRelayHandler$MyByteArrayOutputStream" />
|
||||
<Method name="getBuf" />
|
||||
<Bug pattern="EI_EXPOSE_REP" />
|
||||
</Match>
|
||||
<Match>
|
||||
<!-- these are probably avoidable, but needs further triage to fix -->
|
||||
<Bug pattern="EI_EXPOSE_REP" />
|
||||
</Match>
|
||||
<Match>
|
||||
<!-- these are probably avoidable, but needs further triage to fix -->
|
||||
<Bug pattern="EI_EXPOSE_REP2" />
|
||||
</Match>
|
||||
</FindBugsFilter>
|
||||
@@ -0,0 +1,56 @@
|
||||
package timely.balancer.configuration;
|
||||
|
||||
import javax.net.ssl.SSLContext;
|
||||
|
||||
import org.apache.curator.framework.CuratorFramework;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.ComponentScan;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
import io.netty.handler.ssl.SslContext;
|
||||
import timely.balancer.HealthChecker;
|
||||
import timely.balancer.MetricResolver;
|
||||
import timely.balancer.test.TestBalancer;
|
||||
import timely.balancer.test.TestHealthChecker;
|
||||
import timely.balancer.test.TestMetricResolver;
|
||||
import timely.common.component.AuthenticationService;
|
||||
import timely.common.configuration.CacheProperties;
|
||||
import timely.common.configuration.SecurityProperties;
|
||||
import timely.common.configuration.SslClientProperties;
|
||||
import timely.common.configuration.SslServerProperties;
|
||||
import timely.common.configuration.ZookeeperProperties;
|
||||
|
||||
@Configuration
|
||||
@ComponentScan(basePackages = {"timely.balancer", "timely.common", "timely.test"})
|
||||
public class TestBalancerConfiguration {
|
||||
|
||||
@Bean
|
||||
public TestBalancer balancer(ApplicationContext applicationContext, @Qualifier("nettySslContext") SslContext serverSslContext,
|
||||
@Qualifier("outboundJDKSslContext") SSLContext clientSSLContext, AuthenticationService authenticationService, MetricResolver metricResolver,
|
||||
BalancerProperties balancerProperties, ZookeeperProperties zookeeperProperties, SecurityProperties securityProperties,
|
||||
SslServerProperties sslServerProperties, SslClientProperties sslClientProperties, BalancerServerProperties balancerServerProperties,
|
||||
BalancerHttpProperties balancerHttpProperties, BalancerWebsocketProperties balancerWebsocketProperties) {
|
||||
TestBalancer balancer = new TestBalancer(applicationContext, serverSslContext, clientSSLContext, authenticationService, metricResolver,
|
||||
balancerProperties, zookeeperProperties, securityProperties, sslServerProperties, sslClientProperties, balancerServerProperties,
|
||||
balancerHttpProperties, balancerWebsocketProperties);
|
||||
balancer.start();
|
||||
return balancer;
|
||||
}
|
||||
|
||||
@Bean
|
||||
public TestHealthChecker healthChecker(BalancerProperties balancerProperties, BalancerServerProperties balancerServerProperties) {
|
||||
TestHealthChecker healthChecker = new TestHealthChecker(balancerProperties, balancerServerProperties);
|
||||
healthChecker.start();
|
||||
return healthChecker;
|
||||
}
|
||||
|
||||
@Bean
|
||||
public TestMetricResolver metricResolver(CuratorFramework curatorFramework, BalancerProperties balancerProperties, CacheProperties cacheProperties,
|
||||
HealthChecker healthChecker) {
|
||||
TestMetricResolver metricResolver = new TestMetricResolver(curatorFramework, balancerProperties, cacheProperties, healthChecker);
|
||||
metricResolver.start();
|
||||
return metricResolver;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package timely.balancer.configuration;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.net.ServerSocket;
|
||||
import java.nio.file.Files;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.Executors;
|
||||
|
||||
import org.apache.zookeeper.server.ServerConfig;
|
||||
import org.apache.zookeeper.server.ZooKeeperServerMain;
|
||||
import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
import timely.common.configuration.ZookeeperProperties;
|
||||
|
||||
@Configuration
|
||||
@ConditionalOnProperty(name = "timely.in-memory-accumulo.enabled", havingValue = "true")
|
||||
public class TestZookeeperConfiguration {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(TestZookeeperConfiguration.class);
|
||||
private static final int MAX_PORT = 100;
|
||||
private int port;
|
||||
|
||||
private ZookeeperProperties zookeeperProperties;
|
||||
|
||||
@Bean(name = "ZooKeeperServerMain", destroyMethod = "close")
|
||||
public ZooKeeperServerMain zooKeeperServerMain(ZookeeperProperties zookeeperProperties) throws Exception {
|
||||
this.zookeeperProperties = zookeeperProperties;
|
||||
this.port = getNextPort(2181);
|
||||
this.zookeeperProperties.setServers(String.format("localhost:%d", port));
|
||||
|
||||
File tempDir;
|
||||
try {
|
||||
tempDir = Files.createTempDirectory("zookeeper_temp").toFile();
|
||||
tempDir.deleteOnExit();
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Unable to create temp directory for mini accumulo cluster");
|
||||
}
|
||||
|
||||
Properties properties = new Properties();
|
||||
properties.put("dataDir", tempDir.getAbsolutePath() + "/data");
|
||||
properties.put("clientPort", port);
|
||||
properties.put("skipACL", "true");
|
||||
properties.put("leaderServes", "yes");
|
||||
properties.put("4lw.commands.whitelist", "*");
|
||||
properties.put("admin.enableServer", "false");
|
||||
QuorumPeerConfig quorumPeerConfig = new QuorumPeerConfig();
|
||||
quorumPeerConfig.parseProperties(properties);
|
||||
|
||||
ZooKeeperServerMain zookeeper = new ZooKeeperServerMain();
|
||||
final ServerConfig config = new ServerConfig();
|
||||
config.readFrom(quorumPeerConfig);
|
||||
Executors.newSingleThreadExecutor().submit(() -> {
|
||||
try {
|
||||
zookeeper.runFromConfig(config);
|
||||
} catch (Exception e) {
|
||||
log.error(e.getMessage(), e);
|
||||
}
|
||||
});
|
||||
log.info("Started TestZookeeper on port {}", port);
|
||||
return zookeeper;
|
||||
}
|
||||
|
||||
private int getNextPort(int start) {
|
||||
for (int port = start; port < start + MAX_PORT; ++port) {
|
||||
try {
|
||||
new ServerSocket(port).close();
|
||||
return port;
|
||||
} catch (IOException e) {
|
||||
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
package timely.balancer.integration;
|
||||
|
||||
import static timely.Constants.SERVICE_DISCOVERY_PATH;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.apache.curator.framework.CuratorFramework;
|
||||
import org.apache.curator.framework.state.ConnectionState;
|
||||
import org.apache.curator.x.discovery.ServiceCache;
|
||||
import org.apache.curator.x.discovery.ServiceDiscovery;
|
||||
import org.apache.curator.x.discovery.ServiceDiscoveryBuilder;
|
||||
import org.apache.curator.x.discovery.ServiceInstance;
|
||||
import org.apache.curator.x.discovery.ServiceInstanceBuilder;
|
||||
import org.apache.curator.x.discovery.details.ServiceCacheListener;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.experimental.categories.Category;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.test.context.junit4.SpringRunner;
|
||||
|
||||
import timely.ServerDetails;
|
||||
import timely.balancer.connection.TimelyBalancedHost;
|
||||
import timely.balancer.test.TestHealthChecker;
|
||||
import timely.balancer.test.TestMetricResolver;
|
||||
import timely.test.IntegrationTest;
|
||||
import timely.test.TimelyTestRule;
|
||||
|
||||
@Category(IntegrationTest.class)
|
||||
@RunWith(SpringRunner.class)
|
||||
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE)
|
||||
public class MetricResolverIT {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(MetricResolverIT.class);
|
||||
|
||||
@Autowired
|
||||
@Rule
|
||||
public TimelyTestRule testRule;
|
||||
|
||||
@Autowired
|
||||
private TestMetricResolver metricResolver;
|
||||
|
||||
@Autowired
|
||||
private TestHealthChecker healthChecker;
|
||||
|
||||
@Autowired
|
||||
private CuratorFramework curatorFramework;
|
||||
|
||||
private ServiceDiscovery<ServerDetails> discovery;
|
||||
private ServiceCache<ServerDetails> serviceCache;
|
||||
|
||||
@Before
|
||||
public void setup() throws Exception {
|
||||
discovery = ServiceDiscoveryBuilder.builder(ServerDetails.class).client(curatorFramework).basePath(SERVICE_DISCOVERY_PATH).build();
|
||||
discovery.start();
|
||||
serviceCache = discovery.serviceCacheBuilder().name("timely-server").build();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void ServiceRegistrationTest() {
|
||||
LinkedBlockingQueue<List<ServiceInstance<ServerDetails>>> updateQueue = getServiceUpdateQueue(serviceCache);
|
||||
registerService(curatorFramework, TimelyBalancedHost.of("host1", 5242, 5243, 5244, 5245, null));
|
||||
registerService(curatorFramework, TimelyBalancedHost.of("host1", 5342, 5343, 5344, 5345, null));
|
||||
registerService(curatorFramework, TimelyBalancedHost.of("host1", 5442, 5443, 5444, 5445, null));
|
||||
registerService(curatorFramework, TimelyBalancedHost.of("host1", 5542, 5543, 5544, 5545, null));
|
||||
try {
|
||||
List<ServiceInstance<ServerDetails>> currentUpdate;
|
||||
List<ServiceInstance<ServerDetails>> lastUpdate = null;
|
||||
do {
|
||||
currentUpdate = updateQueue.poll(200, TimeUnit.MILLISECONDS);
|
||||
if (currentUpdate != null) {
|
||||
lastUpdate = currentUpdate;
|
||||
}
|
||||
} while (currentUpdate != null);
|
||||
Assert.assertNotNull(lastUpdate);
|
||||
Assert.assertEquals(4, lastUpdate.size());
|
||||
} catch (InterruptedException e) {
|
||||
Assert.fail("" + e.getMessage());
|
||||
}
|
||||
Assert.assertEquals(4, metricResolver.getServerList().size());
|
||||
Assert.assertEquals(4, healthChecker.getTimelyHosts().size());
|
||||
}
|
||||
|
||||
public void registerService(CuratorFramework curatorFramework, TimelyBalancedHost server) {
|
||||
try {
|
||||
ServerDetails payload = new ServerDetails();
|
||||
payload.setHost(server.getHost());
|
||||
payload.setTcpPort(server.getTcpPort());
|
||||
payload.setHttpPort(server.getHttpPort());
|
||||
payload.setWsPort(server.getWsPort());
|
||||
payload.setUdpPort(server.getUdpPort());
|
||||
|
||||
ServiceInstanceBuilder<ServerDetails> builder = ServiceInstance.builder();
|
||||
String serviceName = server.getHost() + ":" + server.getTcpPort();
|
||||
ServiceInstance<ServerDetails> serviceInstance = builder.id(serviceName).name("timely-server").address(server.getHost()).port(server.getTcpPort())
|
||||
.payload(payload).build();
|
||||
|
||||
ServiceDiscovery<ServerDetails> discovery = ServiceDiscoveryBuilder.builder(ServerDetails.class).client(curatorFramework)
|
||||
.basePath(SERVICE_DISCOVERY_PATH).build();
|
||||
discovery.start();
|
||||
discovery.registerService(serviceInstance);
|
||||
} catch (Exception e) {
|
||||
log.error(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private LinkedBlockingQueue<List<ServiceInstance<ServerDetails>>> getServiceUpdateQueue(ServiceCache<ServerDetails> serviceCache) {
|
||||
|
||||
LinkedBlockingQueue<List<ServiceInstance<ServerDetails>>> updates = new LinkedBlockingQueue<>();
|
||||
try {
|
||||
|
||||
ServiceCacheListener listener = new ServiceCacheListener() {
|
||||
|
||||
@Override
|
||||
public void stateChanged(CuratorFramework curatorFramework, ConnectionState connectionState) {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void cacheChanged() {
|
||||
updates.add(serviceCache.getInstances());
|
||||
}
|
||||
};
|
||||
serviceCache.addListener(listener);
|
||||
serviceCache.start();
|
||||
} catch (Exception e) {
|
||||
log.error(e.getMessage(), e);
|
||||
}
|
||||
return updates;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package timely.balancer.test;
|
||||
|
||||
import javax.net.ssl.SSLContext;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
|
||||
import io.netty.handler.ssl.SslContext;
|
||||
import timely.balancer.Balancer;
|
||||
import timely.balancer.MetricResolver;
|
||||
import timely.balancer.configuration.BalancerHttpProperties;
|
||||
import timely.balancer.configuration.BalancerProperties;
|
||||
import timely.balancer.configuration.BalancerServerProperties;
|
||||
import timely.balancer.configuration.BalancerWebsocketProperties;
|
||||
import timely.common.component.AuthenticationService;
|
||||
import timely.common.configuration.SecurityProperties;
|
||||
import timely.common.configuration.SslClientProperties;
|
||||
import timely.common.configuration.SslServerProperties;
|
||||
import timely.common.configuration.ZookeeperProperties;
|
||||
|
||||
public class TestBalancer extends Balancer {
|
||||
private static final Logger log = LoggerFactory.getLogger(TestBalancer.class);
|
||||
|
||||
public TestBalancer(ApplicationContext applicationContext, SslContext serverSslContext, SSLContext clientSSLContext,
|
||||
AuthenticationService authenticationService, MetricResolver metricResolver, BalancerProperties balancerProperties,
|
||||
ZookeeperProperties zookeeperProperties, SecurityProperties securityProperties, SslServerProperties sslServerProperties,
|
||||
SslClientProperties sslClientProperties, BalancerServerProperties balancerServerProperties, BalancerHttpProperties balancerHttpProperties,
|
||||
BalancerWebsocketProperties balancerWebsocketProperties) {
|
||||
super(applicationContext, serverSslContext, clientSSLContext, authenticationService, metricResolver, balancerProperties, zookeeperProperties,
|
||||
securityProperties, sslServerProperties, sslClientProperties, balancerServerProperties, balancerHttpProperties,
|
||||
balancerWebsocketProperties);
|
||||
DEFAULT_EVENT_LOOP_THREADS = 1;
|
||||
log.info("Starting TestBalancer");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package timely.balancer.test;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import timely.balancer.HealthChecker;
|
||||
import timely.balancer.configuration.BalancerProperties;
|
||||
import timely.balancer.configuration.BalancerServerProperties;
|
||||
import timely.balancer.connection.TimelyBalancedHost;
|
||||
|
||||
public class TestHealthChecker extends HealthChecker {
|
||||
|
||||
public TestHealthChecker(BalancerProperties balancerProperties, BalancerServerProperties balancerServerProperties) {
|
||||
super(balancerProperties, balancerServerProperties);
|
||||
}
|
||||
|
||||
public List<TimelyBalancedHost> getTimelyHosts() {
|
||||
return this.timelyHosts;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package timely.balancer.test;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.curator.framework.CuratorFramework;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import timely.balancer.HealthChecker;
|
||||
import timely.balancer.MetricResolver;
|
||||
import timely.balancer.configuration.BalancerProperties;
|
||||
import timely.balancer.connection.TimelyBalancedHost;
|
||||
import timely.common.configuration.CacheProperties;
|
||||
|
||||
public class TestMetricResolver extends MetricResolver {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(TestMetricResolver.class);
|
||||
|
||||
public TestMetricResolver(CuratorFramework curatorFramework, BalancerProperties balancerProperties, CacheProperties cacheProperties,
|
||||
HealthChecker healthChecker) {
|
||||
super(curatorFramework, balancerProperties, cacheProperties, healthChecker);
|
||||
}
|
||||
|
||||
protected void createAssignmentFile() {
|
||||
try {
|
||||
log.info("Creating assignment file: " + balancerProperties.getAssignmentFile());
|
||||
Configuration configuration = new Configuration();
|
||||
FileSystem fs = FileSystem.get(configuration);
|
||||
Path assignmentFile = new Path(balancerProperties.getAssignmentFile());
|
||||
fs.setWriteChecksum(false);
|
||||
fs.create(assignmentFile);
|
||||
fs.deleteOnExit(assignmentFile);
|
||||
} catch (Exception e) {
|
||||
log.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
public List<TimelyBalancedHost> getServerList() {
|
||||
return this.serverList;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package timely.balancer.test;
|
||||
|
||||
import java.util.Collections;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.context.annotation.Primary;
|
||||
import org.springframework.security.authentication.ProviderManager;
|
||||
import org.springframework.security.web.authentication.preauth.PreAuthenticatedAuthenticationProvider;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Primary
|
||||
@Component
|
||||
public class TestTimelyAuthenticationManager extends ProviderManager {
|
||||
|
||||
@Autowired
|
||||
public TestTimelyAuthenticationManager(@Qualifier("file") PreAuthenticatedAuthenticationProvider file) {
|
||||
super(Collections.singletonList(file));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
logging:
|
||||
level:
|
||||
ROOT: info
|
||||
timely: debug
|
||||
org.apache.zookeeper: error
|
||||
|
||||
timely:
|
||||
test: true
|
||||
instance: 1
|
||||
accumulo:
|
||||
instance-name: default
|
||||
username: root
|
||||
password: secret
|
||||
mini-accumulo:
|
||||
enabled: true
|
||||
rootAuths: A,B,C,D,E,F,G,H,I
|
||||
zookeeper:
|
||||
servers: localhost:0
|
||||
timeout: 30s
|
||||
security:
|
||||
session-max-age: 86400
|
||||
cache-expiration-minutes: 60
|
||||
cache-refresh-minutes: 5
|
||||
allow-anonymous-http-access: true
|
||||
allow-anonymous-ws-access: true
|
||||
authorized-users:
|
||||
- subject-dn: cn=example.com
|
||||
roles: G,H,I
|
||||
auths: D,E,F
|
||||
user-type: USER
|
||||
- subject-dn: cn=test a. user, ou=example developers, o=example corp, c=us
|
||||
issuer-dn: cn=example corp ca, o=example corp, c=us
|
||||
roles: G,H,I
|
||||
auths: A,B,C
|
||||
user-type: USER
|
||||
server-ssl:
|
||||
key-store-type: PKCS12
|
||||
key-store-file: classpath:ssl/testServer.p12
|
||||
key-store-password: ChangeIt
|
||||
trust-store-type: JKS
|
||||
trust-store-file: classpath:ssl/ca.jks
|
||||
trust-store-password: ChangeIt
|
||||
use-generated-keypair: false
|
||||
ignore-ssl-handshake-errors: true
|
||||
use-openssl: true
|
||||
client-ssl:
|
||||
key-store-type: PKCS12
|
||||
key-store-file: classpath:ssl/testUser.p12
|
||||
key-store-password: ChangeIt
|
||||
trust-store-type: JKS
|
||||
trust-store-file: classpath:ssl/ca.jks
|
||||
trust-store-password: ChangeIt
|
||||
cache:
|
||||
enabled: false
|
||||
metric-age-off-hours:
|
||||
default: 26
|
||||
non-cached-metrics:
|
||||
balancer:
|
||||
check-server-health-interval: 10000
|
||||
server-failures-before-down: 6
|
||||
server-successes-before-up: 1
|
||||
assignment-file: file://${java.io.tmpdir}/timely/balancer/assignments.csv
|
||||
server:
|
||||
ip: 127.0.0.1
|
||||
tcp-base-port: 4242
|
||||
num-tcp-pools: 1
|
||||
tcp-buffer-size: 100
|
||||
tcp-client-pool:
|
||||
min-idle-per-key: 0
|
||||
max-idle-per-key: 40
|
||||
max-total-per-key: 40
|
||||
max-total: -1
|
||||
lifo: true
|
||||
fairness: false
|
||||
max-wait-millis: -1
|
||||
min-evictable-idle-time-millis: 1800000
|
||||
evictor-shutdown-timeout-millis: 10000
|
||||
soft-min-evictable-idle-time-millis: -1
|
||||
num-tests-per-eviction-run: 3
|
||||
time-between-eviction-runs-millis: -1
|
||||
block-when-exhausted: true
|
||||
jmx-enabled: true
|
||||
test-on-borrow: false
|
||||
test-on-return: false
|
||||
udp-base-port: 4245
|
||||
udp-client-pool:
|
||||
min-idle-per-key: 0
|
||||
max-idle-per-key: 40
|
||||
max-total-per-key: 40
|
||||
max-total: -1
|
||||
lifo: true
|
||||
fairness: false
|
||||
max-wait-millis: -1
|
||||
min-evictable-idle-time-millis: 1800000
|
||||
evictor-shutdown-timeout-millis: 10000
|
||||
soft-min-evictable-idle-time-millis: -1
|
||||
num-tests-per-eviction-run: 3
|
||||
time-between-eviction-runs-millis: -1
|
||||
block-when-exhausted: true
|
||||
jmx-enabled: true
|
||||
test-on-borrow: false
|
||||
test-on-return: false
|
||||
http:
|
||||
host: localhost
|
||||
ip: 127.0.0.1
|
||||
base-port: 4243
|
||||
redirect-path: /secure-me
|
||||
strict-transport-max-age: 604800
|
||||
cors:
|
||||
allow-any-origin: false
|
||||
allow-null-origin: false
|
||||
allow-credentials: true
|
||||
allowed-origins:
|
||||
allowed-methods: DELETE,GET,HEAD,OPTIONS,PUT,POST
|
||||
allowed-headers: content-type
|
||||
http-client-pool:
|
||||
min-idle-per-key: 0
|
||||
max-idle-per-key: 40
|
||||
max-total-per-key: 40
|
||||
max-total: -1
|
||||
lifo: true
|
||||
fairness: false
|
||||
max-wait-millis: -1
|
||||
min-evictable-idle-time-millis: 1800000
|
||||
evictor-shutdown-timeout-millis: 10000
|
||||
soft-min-evictable-idle-time-millis: -1
|
||||
num-tests-per-eviction-run: 3
|
||||
time-between-eviction-runs-millis: -1
|
||||
block-when-exhausted: true
|
||||
jmx-enabled: true
|
||||
test-on-borrow: false
|
||||
test-on-return: false
|
||||
websocket:
|
||||
ip: 127.0.0.1
|
||||
base-port: 5244
|
||||
timeout: 60
|
||||
subscription-lag: 120
|
||||
scanner-batch-size: 10000
|
||||
flush-interval-seconds: 30
|
||||
scanner-read-ahead: 1
|
||||
subscription-batch-size: 5000
|
||||
incoming-buffer-size: 50000000
|
||||
ws-client-pool:
|
||||
min-idle-per-key: 0
|
||||
max-idle-per-key: 40
|
||||
max-total-per-key: 40
|
||||
max-total: -1
|
||||
lifo: true
|
||||
fairness: false
|
||||
max-wait-millis: -1
|
||||
min-evictable-idle-time-millis: 1800000
|
||||
evictor-shutdown-timeout-millis: 10000
|
||||
soft-min-evictable-idle-time-millis: -1
|
||||
num-tests-per-eviction-run: 3
|
||||
time-between-eviction-runs-millis: -1
|
||||
block-when-exhausted: true
|
||||
jmx-enabled: true
|
||||
test-on-borrow: false
|
||||
test-on-return: false
|
||||
@@ -0,0 +1,3 @@
|
||||
spring:
|
||||
application:
|
||||
name: timely-balancer
|
||||
@@ -0,0 +1,24 @@
|
||||
Configuration:
|
||||
status: warn
|
||||
monitorInterval: 60
|
||||
|
||||
Properties:
|
||||
Property:
|
||||
- name: PID
|
||||
value: "????"
|
||||
- name: LOG_PATTERN
|
||||
value: "%clr{%d{yyyy-MM-dd HH:mm:ss.SSS}}{faint} %clr{%5p} %clr{${sys:PID}}{magenta} %clr{---}{faint} %clr{[%15.15t]}{faint} %clr{%-40.40c{1.}}{cyan} %clr{:}{faint} %m%n%wEx"
|
||||
|
||||
Appenders:
|
||||
Console:
|
||||
name: Console
|
||||
target: SYSTEM_OUT
|
||||
follow: true
|
||||
PatternLayout:
|
||||
pattern: "${LOG_PATTERN}"
|
||||
|
||||
Loggers:
|
||||
Root:
|
||||
level: warn
|
||||
AppenderRef:
|
||||
- ref: Console
|
||||
@@ -0,0 +1,23 @@
|
||||
Bag Attributes: <No Attributes>
|
||||
subject=/C=US/O=Example Corp/CN=EXAMPLE CORP CA
|
||||
issuer=/C=US/O=Example Corp/CN=EXAMPLE CORP CA
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIDTjCCAjagAwIBAgIURUj8l+ms/i9jeabIBBi19KxEhAgwDQYJKoZIhvcNAQEL
|
||||
BQAwPjELMAkGA1UEBhMCVVMxFTATBgNVBAoTDEV4YW1wbGUgQ29ycDEYMBYGA1UE
|
||||
AxMPRVhBTVBMRSBDT1JQIENBMCAXDTIzMDIwMTIzMTUwMFoYDzIxMjMwMTA4MjMx
|
||||
NTAwWjA+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRXhhbXBsZSBDb3JwMRgwFgYD
|
||||
VQQDEw9FWEFNUExFIENPUlAgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
|
||||
AoIBAQDVBr2XF20SKmGXfvp2BUY44BF1fm3Qw83GkC66qIHgpgQxzJ8u44P34vV1
|
||||
DYTj4XzrBpx23J0nCMEiAW8YW9MxZJPEokZafRNKFQAngVdgmAW9fHemHRQfaYwe
|
||||
ySVz2UZ+z28DJ3C0QshvqXskrYstwLVs4Ss2uoX+XpeFbW9IVVg4aW+GX3LHPNTk
|
||||
7Hji13Capp6BCiS61AGQAiMWNBW2aRgHVLOb7RDQug2Bka/dcJ44XxJDh26C6kV1
|
||||
pNEZm0ToqWAiVfvyzionPec5im6NKX9kStZc+C/iikfx5vdmrvGIF5zxy6phdhJ7
|
||||
AGd6cF6UBkkYm/SsIP3WTXpBek5LAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAP
|
||||
BgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSPLNlCmmp9PH+crzCx9TNp5BoCajAN
|
||||
BgkqhkiG9w0BAQsFAAOCAQEAkjLnZKqLMjbu5Hy8YOVoKaEKahP+t/tNYZW/NtrC
|
||||
xSxKmGLz5kVCkjpLgCykkcpu4h2wZW4MGZJ4nc87+tN22mJOg1WnqEpBzRsEPajk
|
||||
7duVjdU2FDCarERj53hxF41l/u8BxvMGbFnrAijv5SzspeLi+VCGF3wu53Bi6UHi
|
||||
7WO1dIhZLMHepqgtwbDDudxwlKbG/JWjJ7OspCNqOpktpY8BFUAQ42ctYZcOHy2w
|
||||
kdZDcK5IhRztkBH2aSWOhudYh/CF7BOu2nSUdtR5NwhXnsStmSbud7ahw3W5Jc9/
|
||||
IVAFXf2brkrfIzwrY5kmjnEPJ8mjUyLepwSlOlMTBrehzQ==
|
||||
-----END CERTIFICATE-----
|
||||
Binary file not shown.
@@ -0,0 +1,51 @@
|
||||
Bag Attributes
|
||||
localKeyID: 22 89 1C EA 4E FF 9A F6 11 1B E7 9B 5E 4F 59 B7 B1 0F AA FA
|
||||
friendlyName: example-server
|
||||
subject=/C=US/O=Example Corp/OU=Servers/CN=testserver.example.com
|
||||
issuer=/C=US/O=Example Corp/CN=EXAMPLE CORP CA
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIDwjCCAqqgAwIBAgIUe6yBgkY1m/oXuPXA9j/c/7+w16UwDQYJKoZIhvcNAQEL
|
||||
BQAwPjELMAkGA1UEBhMCVVMxFTATBgNVBAoTDEV4YW1wbGUgQ29ycDEYMBYGA1UE
|
||||
AxMPRVhBTVBMRSBDT1JQIENBMCAXDTIzMDIwMTIzMTUwMFoYDzIxMjMwMTA4MjMx
|
||||
NTAwWjBXMQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRXhhbXBsZSBDb3JwMRAwDgYD
|
||||
VQQLEwdTZXJ2ZXJzMR8wHQYDVQQDExZ0ZXN0c2VydmVyLmV4YW1wbGUuY29tMIIB
|
||||
IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAmb1pEWg9a6ZChQULazK75ppc
|
||||
XScU5Ap29L70/5zcaOcbaBwqKhpKhdZCoZ1TD+Sdkk9kp7IlabCQjM2vVjknb6NY
|
||||
msl3PbQFNva0EkH6s5cvpX4WhQDsWK77/4eztHT+0fpaG1YoKQn+Gy3VuW0CDyR6
|
||||
GzGKifnI/aNLevgUBebbiiTPLEdGlo0jp3SXlZfoDZdbvnhBWI8Xc+aDekNlrXxW
|
||||
X2TVnc02Fi914PECYJzkMpmxnBRPAcljU24WSqaUVgRoSbafkm4OXDw0xtvxnIrJ
|
||||
s3iO0uZfRf7+OxAlqlvh4ZJ9/PufPMEKvmr6zea05KkFjBotKMip5Md43H3lQQID
|
||||
AQABo4GcMIGZMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYI
|
||||
KwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUDWwMrl9/uc1wplERWxas
|
||||
bDROB4MwHwYDVR0jBBgwFoAUjyzZQppqfTx/nK8wsfUzaeQaAmowGgYDVR0RBBMw
|
||||
EYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEBCwUAA4IBAQBfG7StgQf3WNk/
|
||||
EnaYuurwvaOOpcOV5NDcZ6tmgaXUrISPVDpyIWfsDGetNEXc+4gpefLQtGF2wuiW
|
||||
odiLc4H9JwQ/yEkVYwzrFViPdZKVvxNU2EjiwO/e/3sl+SxxkTDy6o838uTXX+X5
|
||||
Q8DabmL5L8ukcnqUHVOqW38oA1vRNBKiJX58JzOr8ezX8crZmqU9h8cNwG/V+ZXE
|
||||
5U7+hSRzLm7gRwtixj8ZsTDUiGJxOdbyUKImPClt1Se9TizB2Q/xOPTdmYB1qiiB
|
||||
+XVKkGwQPgIbHckLsZWhBor2u5cgdrBGW09Q+H88maagwq5sBHNMs33vkkgbUDpB
|
||||
6WSEBJMi
|
||||
-----END CERTIFICATE-----
|
||||
Bag Attributes: <No Attributes>
|
||||
subject=/C=US/O=Example Corp/CN=EXAMPLE CORP CA
|
||||
issuer=/C=US/O=Example Corp/CN=EXAMPLE CORP CA
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIDTjCCAjagAwIBAgIURUj8l+ms/i9jeabIBBi19KxEhAgwDQYJKoZIhvcNAQEL
|
||||
BQAwPjELMAkGA1UEBhMCVVMxFTATBgNVBAoTDEV4YW1wbGUgQ29ycDEYMBYGA1UE
|
||||
AxMPRVhBTVBMRSBDT1JQIENBMCAXDTIzMDIwMTIzMTUwMFoYDzIxMjMwMTA4MjMx
|
||||
NTAwWjA+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRXhhbXBsZSBDb3JwMRgwFgYD
|
||||
VQQDEw9FWEFNUExFIENPUlAgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
|
||||
AoIBAQDVBr2XF20SKmGXfvp2BUY44BF1fm3Qw83GkC66qIHgpgQxzJ8u44P34vV1
|
||||
DYTj4XzrBpx23J0nCMEiAW8YW9MxZJPEokZafRNKFQAngVdgmAW9fHemHRQfaYwe
|
||||
ySVz2UZ+z28DJ3C0QshvqXskrYstwLVs4Ss2uoX+XpeFbW9IVVg4aW+GX3LHPNTk
|
||||
7Hji13Capp6BCiS61AGQAiMWNBW2aRgHVLOb7RDQug2Bka/dcJ44XxJDh26C6kV1
|
||||
pNEZm0ToqWAiVfvyzionPec5im6NKX9kStZc+C/iikfx5vdmrvGIF5zxy6phdhJ7
|
||||
AGd6cF6UBkkYm/SsIP3WTXpBek5LAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAP
|
||||
BgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSPLNlCmmp9PH+crzCx9TNp5BoCajAN
|
||||
BgkqhkiG9w0BAQsFAAOCAQEAkjLnZKqLMjbu5Hy8YOVoKaEKahP+t/tNYZW/NtrC
|
||||
xSxKmGLz5kVCkjpLgCykkcpu4h2wZW4MGZJ4nc87+tN22mJOg1WnqEpBzRsEPajk
|
||||
7duVjdU2FDCarERj53hxF41l/u8BxvMGbFnrAijv5SzspeLi+VCGF3wu53Bi6UHi
|
||||
7WO1dIhZLMHepqgtwbDDudxwlKbG/JWjJ7OspCNqOpktpY8BFUAQ42ctYZcOHy2w
|
||||
kdZDcK5IhRztkBH2aSWOhudYh/CF7BOu2nSUdtR5NwhXnsStmSbud7ahw3W5Jc9/
|
||||
IVAFXf2brkrfIzwrY5kmjnEPJ8mjUyLepwSlOlMTBrehzQ==
|
||||
-----END CERTIFICATE-----
|
||||
@@ -0,0 +1,32 @@
|
||||
Bag Attributes
|
||||
localKeyID: 22 89 1C EA 4E FF 9A F6 11 1B E7 9B 5E 4F 59 B7 B1 0F AA FA
|
||||
friendlyName: example-server
|
||||
Key Attributes: <No Attributes>
|
||||
-----BEGIN PRIVATE KEY-----
|
||||
MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCZvWkRaD1rpkKF
|
||||
BQtrMrvmmlxdJxTkCnb0vvT/nNxo5xtoHCoqGkqF1kKhnVMP5J2ST2SnsiVpsJCM
|
||||
za9WOSdvo1iayXc9tAU29rQSQfqzly+lfhaFAOxYrvv/h7O0dP7R+lobVigpCf4b
|
||||
LdW5bQIPJHobMYqJ+cj9o0t6+BQF5tuKJM8sR0aWjSOndJeVl+gNl1u+eEFYjxdz
|
||||
5oN6Q2WtfFZfZNWdzTYWL3Xg8QJgnOQymbGcFE8ByWNTbhZKppRWBGhJtp+Sbg5c
|
||||
PDTG2/GcismzeI7S5l9F/v47ECWqW+Hhkn38+588wQq+avrN5rTkqQWMGi0oyKnk
|
||||
x3jcfeVBAgMBAAECggEAZEHC7NkfhL4NjxV1Ftkk9AMhzJWZ6ARzArQsrSysB7T+
|
||||
iGwliwNOKe1J0n74FdsubqVbrIheIYhFG61edEodT44kT/ylyBVHeUp0r5tbVBZ5
|
||||
ueB56w5pEoQ6Ta8/H6VUL7alYEJVOj4L+clwINXXi+xrvKQYBlZgHheJN43j8NxS
|
||||
9lCrVcm58rzX3i29S7ErBf7RqDabgD6j2m7rehnSq5Ps2wxwV/LXMl4Lxlwa0jz/
|
||||
LV8szNMZViPOWAqOCy7NZbwCCaCfKVUAwmNGBkE3p4JVPeLNLzCBY9G0rHVmFJLK
|
||||
yPnCB7PtCZeFTsU+8OHHWoxMFsaOO9Pp8FtB/r8PYQKBgQDI0OzEo3t/QKvbwAdz
|
||||
9Eze0LQvstzZIidTWbYMTxH0g7DIt+3p11d61GKtfeuYUTPlLodufdv/BaP8G9sI
|
||||
NzxisB3uH/787M9fwi1OzO7zMMZZCVa7Q1vZk6ta1qY3PekcYMPqjBJhxP1X3S7x
|
||||
8+mrW5vTq/l8ZRVFeLvEkh6bpQKBgQDD/MCRJoq6nzd2DoTyT9m5qO+Jlztmu9+T
|
||||
vqVOg7q0NO7U7dzyMbWgMvMFoEBidnyllU1rEj2YOo4OrI2v/JmSPUVS8VXsBkkG
|
||||
UqInV4F/g0Wp3hzrkZFMqHLa6AydugmRNkOpgurRKOiwAL0Pgvk8+bOIbKMM7JBE
|
||||
tZ6Zvi0gbQKBgD4S5r0505jYQsdgqm0bXQ+a77e20DGGLyXCm1Ves6wjcOf6gt4/
|
||||
pFjhgccjH0or/6CeK1rqKLDB65ioe6IkgNP4oKfU2BsSkF+HRD4RduNRdl5bVqX4
|
||||
AlJVJSilJSf6SS5ucuDzf1MdczpPso0z2k0PujFJ/YqILHUXBuOST0fJAoGAZAdX
|
||||
nX3HZpKmsumcLzRUEq9HerkmRzvyU60FH/qZyyoND75Le42lfpsicCenVNnWqoh2
|
||||
WASaTVztbk3hjwXkt7/v5vJoASwAD2sTefLulE9Cuf00U4MbXr3GyTWQYtw1z9ib
|
||||
ViQMhy5S/KmT1Rvq65UibanM2mbPB1KPHTb0P00CgYEAgieJ1YXYYHPjABWJ7/S9
|
||||
3Z/8eJY0kuUe1DRFaKOlfej8SOCFFIa90IW2mAnujh55H4N2lww4s/N+FIRQoQpA
|
||||
1Hrch9tYRiXyouQc67dFlLxmCOVJIf31ZUt7I4HcPhsfr4fuCJXUPBs1jaDmwQsk
|
||||
90VWrXaTwSjmP+yHMiPfaA0=
|
||||
-----END PRIVATE KEY-----
|
||||
Binary file not shown.
@@ -0,0 +1,50 @@
|
||||
Bag Attributes
|
||||
localKeyID: 57 F6 9F DF B2 94 61 47 B0 B0 2E 3A F7 79 98 C8 15 5C 64 FA
|
||||
friendlyName: example-client
|
||||
subject=/C=US/O=Example Corp/OU=Example Developers/CN=Test A. User
|
||||
issuer=/C=US/O=Example Corp/CN=EXAMPLE CORP CA
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIDmzCCAoOgAwIBAgIUFp4IuIaZ2XId7UNvzP6ZB9hChyUwDQYJKoZIhvcNAQEL
|
||||
BQAwPjELMAkGA1UEBhMCVVMxFTATBgNVBAoTDEV4YW1wbGUgQ29ycDEYMBYGA1UE
|
||||
AxMPRVhBTVBMRSBDT1JQIENBMCAXDTIzMDIwMTIzMTUwMFoYDzIxMjMwMTA4MjMx
|
||||
NTAwWjBYMQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRXhhbXBsZSBDb3JwMRswGQYD
|
||||
VQQLExJFeGFtcGxlIERldmVsb3BlcnMxFTATBgNVBAMTDFRlc3QgQS4gVXNlcjCC
|
||||
ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMOMeN0ElMDxYKST2VD1wP6Z
|
||||
jmy543Zv/QgRLTYizg3U9t3uZU0C3HqUJnK+c4+OJnRIyHuMXuJWEYVdairyAdYW
|
||||
ZDF5sfiL9ikG9rZ2WxR2z7nsYLJcDGwKAt/Duzuoed6KSccEpiQc+BkoOTs9p87I
|
||||
l4s7saX/anESMUY7YOomLSqS2zvQZh/zTjIdP1/9kG9PLzXPClJELuiC3pJCmr7Y
|
||||
xdJIXecu6YKFpApy0y+rKcfYUVPML15gyRu9pIqH3U39y7mCkn+YP9Qc9GcZAoqR
|
||||
iYdUEl1rFAgV/a3o0pmf1tFFRDOsoPSMUGNNQlaCjv1uZIbRm25nvvQlPfstFDEC
|
||||
AwEAAaN1MHMwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMCMAwG
|
||||
A1UdEwEB/wQCMAAwHQYDVR0OBBYEFPAe3n59DcNwRcnpIwQY2kPawbCVMB8GA1Ud
|
||||
IwQYMBaAFI8s2UKaan08f5yvMLH1M2nkGgJqMA0GCSqGSIb3DQEBCwUAA4IBAQBo
|
||||
+zadU9DwR1rnHTBZF6CJXmiIdrQmArojH4a3NOHbNjFjRXm7dkd/tAy5LtgDJkww
|
||||
yg+ZrOyOKHbFgk6IXRZ8nw2b1Ck9I9MDW1vxfjid9YLW2nhDaMcIeTXz3aIBO5i6
|
||||
gMCV+xMcFaU/O57Ohia+ADUqrqu14nG9Moi171UWPbMr5N3gYn3Fs4VrTGYs4i0Z
|
||||
5x2itm88WB3bWj+cd06zKQopU9b0Y76mQv9h+w4EHxq4uWA7GmIDDkvtI61MQo8+
|
||||
twdX159Ms6o4lW43SRqVBHeMgSJbiLDkdkypYJvd9PvAjWgmQK/sD1SYKxfgYhtx
|
||||
hqIvAjlGeDwdf+55EGjf
|
||||
-----END CERTIFICATE-----
|
||||
Bag Attributes: <No Attributes>
|
||||
subject=/C=US/O=Example Corp/CN=EXAMPLE CORP CA
|
||||
issuer=/C=US/O=Example Corp/CN=EXAMPLE CORP CA
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIDTjCCAjagAwIBAgIURUj8l+ms/i9jeabIBBi19KxEhAgwDQYJKoZIhvcNAQEL
|
||||
BQAwPjELMAkGA1UEBhMCVVMxFTATBgNVBAoTDEV4YW1wbGUgQ29ycDEYMBYGA1UE
|
||||
AxMPRVhBTVBMRSBDT1JQIENBMCAXDTIzMDIwMTIzMTUwMFoYDzIxMjMwMTA4MjMx
|
||||
NTAwWjA+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRXhhbXBsZSBDb3JwMRgwFgYD
|
||||
VQQDEw9FWEFNUExFIENPUlAgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
|
||||
AoIBAQDVBr2XF20SKmGXfvp2BUY44BF1fm3Qw83GkC66qIHgpgQxzJ8u44P34vV1
|
||||
DYTj4XzrBpx23J0nCMEiAW8YW9MxZJPEokZafRNKFQAngVdgmAW9fHemHRQfaYwe
|
||||
ySVz2UZ+z28DJ3C0QshvqXskrYstwLVs4Ss2uoX+XpeFbW9IVVg4aW+GX3LHPNTk
|
||||
7Hji13Capp6BCiS61AGQAiMWNBW2aRgHVLOb7RDQug2Bka/dcJ44XxJDh26C6kV1
|
||||
pNEZm0ToqWAiVfvyzionPec5im6NKX9kStZc+C/iikfx5vdmrvGIF5zxy6phdhJ7
|
||||
AGd6cF6UBkkYm/SsIP3WTXpBek5LAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAP
|
||||
BgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSPLNlCmmp9PH+crzCx9TNp5BoCajAN
|
||||
BgkqhkiG9w0BAQsFAAOCAQEAkjLnZKqLMjbu5Hy8YOVoKaEKahP+t/tNYZW/NtrC
|
||||
xSxKmGLz5kVCkjpLgCykkcpu4h2wZW4MGZJ4nc87+tN22mJOg1WnqEpBzRsEPajk
|
||||
7duVjdU2FDCarERj53hxF41l/u8BxvMGbFnrAijv5SzspeLi+VCGF3wu53Bi6UHi
|
||||
7WO1dIhZLMHepqgtwbDDudxwlKbG/JWjJ7OspCNqOpktpY8BFUAQ42ctYZcOHy2w
|
||||
kdZDcK5IhRztkBH2aSWOhudYh/CF7BOu2nSUdtR5NwhXnsStmSbud7ahw3W5Jc9/
|
||||
IVAFXf2brkrfIzwrY5kmjnEPJ8mjUyLepwSlOlMTBrehzQ==
|
||||
-----END CERTIFICATE-----
|
||||
@@ -0,0 +1,32 @@
|
||||
Bag Attributes
|
||||
localKeyID: 57 F6 9F DF B2 94 61 47 B0 B0 2E 3A F7 79 98 C8 15 5C 64 FA
|
||||
friendlyName: example-client
|
||||
Key Attributes: <No Attributes>
|
||||
-----BEGIN PRIVATE KEY-----
|
||||
MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDDjHjdBJTA8WCk
|
||||
k9lQ9cD+mY5sueN2b/0IES02Is4N1Pbd7mVNAtx6lCZyvnOPjiZ0SMh7jF7iVhGF
|
||||
XWoq8gHWFmQxebH4i/YpBva2dlsUds+57GCyXAxsCgLfw7s7qHneiknHBKYkHPgZ
|
||||
KDk7PafOyJeLO7Gl/2pxEjFGO2DqJi0qkts70GYf804yHT9f/ZBvTy81zwpSRC7o
|
||||
gt6SQpq+2MXSSF3nLumChaQKctMvqynH2FFTzC9eYMkbvaSKh91N/cu5gpJ/mD/U
|
||||
HPRnGQKKkYmHVBJdaxQIFf2t6NKZn9bRRUQzrKD0jFBjTUJWgo79bmSG0ZtuZ770
|
||||
JT37LRQxAgMBAAECggEBALFWsT1vLCAWRxi55Hkhhhbe12UO55O/V+969E+GHOw6
|
||||
WG9IBaJjl+Oqe3vP8CkeNw15/CbfVCVa/KS/5Mpxkuy+fbrIrHfFJkZqxE8Q4uSa
|
||||
IyRvk/nstG8+XQDlFuWxkmQZcXMFak6R3wWhjtWEwqwrChQA3Mi366YDY6BXtTpi
|
||||
MuLJVWXqYVF2RirWX3a7eosR4MYBCJ1S8NnpI1w4oZwuew6iYiCJNkURctxnGKOe
|
||||
hoop/711PEyIb/qHETBIlReY9scvZgJQZMgGe5mTGgFmAA8Bzo4e1/F2QrVIi7wy
|
||||
REOSj4U+tP/QrYMSPk0cMpH5ft5+2mfh/o+D1ugm/AkCgYEA2qt3P4vlm1TMmJru
|
||||
vMYc8iZjfe7UQTnD+J1EgrASBI0JP565zSZKBRIXnheumWhWS+2KR1MP8N6KAu4Q
|
||||
xE/E7TKfLuE1CPzn7Rh/WZAxDqGNeQthk0qTbeBZ4/mHPKvt8V7c+N4vLjZ00uul
|
||||
bKitlLpb49T+8FpSgRmk+nrBsesCgYEA5O6LmxH1+ip5NoMHoZVBhLbgJkxoXEzn
|
||||
xHT7On8CjQ9B0T85NW9O38/a2gyIb5dIavcrh4dMvOKX8hdrVlal3chrXFRemyvy
|
||||
VzpW7HZpMwthJb/QSim+GQ4u2Jsm+eXQAswq/yfowls9mp7K81McBgdfcdWgWuqc
|
||||
Ttf3dBRS71MCgYBCPpu7ppgCfKgWuykEC0iNm0+cwROQRv21fX9Z0BlFQ6y/l0hW
|
||||
e/kEnDyHakk8tD0gd2/it6kmoWRSCukNb9UKvsE2pIc6l9NzIm7CJUqXUh3Xw7jl
|
||||
tDwBl2vkAp4TpUpKSyDKTBUi/OBbXYQZ7OfZHLPly4sIKXTBI5G1SCcRzQKBgE3Y
|
||||
Y7ZmRBs+CBumMJkcVLwZJ10+jBoeuIzQLoF0oNxwfLP3aK+n2s02M7gDnP09Fw3w
|
||||
YGfkdQ6hioNyvTgNnLcuqHsr4UOojO5mUxpzJVCLJQyeDei6Cv8gGIPlNjqxj6rz
|
||||
3AcJfhdgyTDjoCVfdF8EcMsXlkB0GJSi0W0yYAUrAoGAVK5v0cbEEV+gxdQgh848
|
||||
FwSQXx7i9jVOB2BUTLX2Q+gLUJibtFzLnh03sGCXtTa8sK+hLW4FCMWZa9Ev5ZjG
|
||||
ylFyA96U97CNpS+2/ZW5+hvRZ/GuY/0yB8w26BeS38lI8RnmxSPi5cnexdl32NIX
|
||||
5FkU9WJ4HKmi1n0304CkF+4=
|
||||
-----END PRIVATE KEY-----
|
||||
Binary file not shown.
@@ -0,0 +1 @@
|
||||
/target/
|
||||
+151
@@ -0,0 +1,151 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>timely</groupId>
|
||||
<artifactId>timely</artifactId>
|
||||
<version>2.1.0</version>
|
||||
</parent>
|
||||
<artifactId>timely-client</artifactId>
|
||||
<properties>
|
||||
<!-- PEP 440 compliant version gets substituted into src/main/python/template/setup.py -->
|
||||
<!-- use .dev0 instead of -SNAPSHOT -->
|
||||
<python.client.version>2.1.0</python.client.version>
|
||||
</properties>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-annotations</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-core</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-databind</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.datatype</groupId>
|
||||
<artifactId>jackson-datatype-jdk8</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
<artifactId>guava</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-lang3</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
<artifactId>httpclient</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.glassfish.tyrus</groupId>
|
||||
<artifactId>tyrus-client</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.glassfish.tyrus</groupId>
|
||||
<artifactId>tyrus-container-grizzly-client</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>timely</groupId>
|
||||
<artifactId>timely-common</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-api</artifactId>
|
||||
<scope>runtime</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-core</artifactId>
|
||||
<scope>runtime</scope>
|
||||
</dependency>
|
||||
<!--
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-slf4j-impl</artifactId>
|
||||
<scope>runtime</scope>
|
||||
</dependency>
|
||||
-->
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-resources-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>filter-setup</id>
|
||||
<goals>
|
||||
<goal>copy-resources</goal>
|
||||
</goals>
|
||||
<phase>process-resources</phase>
|
||||
<configuration>
|
||||
<outputDirectory>${project.basedir}/src/main/python</outputDirectory>
|
||||
<resources>
|
||||
<resource>
|
||||
<directory>${project.basedir}/src/main/python/template</directory>
|
||||
<includes>
|
||||
<include>setup.py</include>
|
||||
</includes>
|
||||
<filtering>true</filtering>
|
||||
</resource>
|
||||
</resources>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>python</id>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.codehaus.mojo</groupId>
|
||||
<artifactId>exec-maven-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>python-dist</id>
|
||||
<goals>
|
||||
<goal>exec</goal>
|
||||
</goals>
|
||||
<phase>package</phase>
|
||||
<configuration>
|
||||
<executable>python</executable>
|
||||
<workingDirectory>${project.basedir}/src/main/python</workingDirectory>
|
||||
<commandlineArgs>setup.py sdist --dist-dir ${project.build.directory}</commandlineArgs>
|
||||
</configuration>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>python-clean</id>
|
||||
<goals>
|
||||
<goal>exec</goal>
|
||||
</goals>
|
||||
<phase>package</phase>
|
||||
<configuration>
|
||||
<executable>rm</executable>
|
||||
<workingDirectory>${project.basedir}/src/main/python</workingDirectory>
|
||||
<commandlineArgs>-rf timely.egg-info setup.py</commandlineArgs>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
</profiles>
|
||||
</project>
|
||||
@@ -0,0 +1,81 @@
|
||||
package timely.api;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
public class AddSubscription extends SubscriptionRequest {
|
||||
|
||||
private String operation = "add";
|
||||
private String metric = null;
|
||||
private Optional<Map<String,String>> tags = Optional.empty();
|
||||
private Optional<Long> startTime = Optional.empty();
|
||||
private Optional<Long> endTime = Optional.empty();
|
||||
private Optional<Long> delayTime = Optional.empty();
|
||||
|
||||
public String getOperation() {
|
||||
return operation;
|
||||
}
|
||||
|
||||
public String getMetric() {
|
||||
return metric;
|
||||
}
|
||||
|
||||
public Optional<Map<String,String>> getTags() {
|
||||
return tags;
|
||||
}
|
||||
|
||||
public Optional<Long> getStartTime() {
|
||||
return startTime;
|
||||
}
|
||||
|
||||
public Optional<Long> getDelayTime() {
|
||||
return delayTime;
|
||||
}
|
||||
|
||||
public void setMetric(String metric) {
|
||||
this.metric = metric;
|
||||
}
|
||||
|
||||
public void setTags(Optional<Map<String,String>> tags) {
|
||||
this.tags = tags;
|
||||
}
|
||||
|
||||
public void setStartTime(Optional<Long> startTime) {
|
||||
this.startTime = startTime;
|
||||
}
|
||||
|
||||
public void setDelayTime(Optional<Long> delayTime) {
|
||||
this.delayTime = delayTime;
|
||||
}
|
||||
|
||||
public Optional<Long> getEndTime() {
|
||||
return endTime;
|
||||
}
|
||||
|
||||
public void setEndTime(Optional<Long> endTime) {
|
||||
this.endTime = endTime;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return (operation + subscriptionId + metric + tags + startTime + endTime + delayTime).hashCode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (null == obj) {
|
||||
return false;
|
||||
}
|
||||
if (obj == this) {
|
||||
return true;
|
||||
}
|
||||
if (obj instanceof AddSubscription) {
|
||||
AddSubscription other = (AddSubscription) obj;
|
||||
return (this.subscriptionId.equals(other.subscriptionId) && this.metric.equals(other.metric) && this.tags.equals(other.tags)
|
||||
&& this.startTime.equals(other.startTime) && this.endTime.equals(other.endTime) && this.delayTime.equals(other.delayTime));
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package timely.api;
|
||||
|
||||
public class CloseSubscription extends SubscriptionRequest {
|
||||
|
||||
private String operation = "close";
|
||||
|
||||
public String getOperation() {
|
||||
return operation;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return (operation + subscriptionId).hashCode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (null == obj) {
|
||||
return false;
|
||||
}
|
||||
if (obj == this) {
|
||||
return true;
|
||||
}
|
||||
if (obj instanceof CloseSubscription) {
|
||||
CloseSubscription other = (CloseSubscription) obj;
|
||||
return (this.subscriptionId.equals(other.subscriptionId));
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package timely.api;
|
||||
|
||||
public class CreateSubscription extends SubscriptionRequest {
|
||||
|
||||
private String operation = "create";
|
||||
|
||||
public String getOperation() {
|
||||
return operation;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return (operation + subscriptionId).hashCode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (null == obj) {
|
||||
return false;
|
||||
}
|
||||
if (obj == this) {
|
||||
return true;
|
||||
}
|
||||
if (obj instanceof CreateSubscription) {
|
||||
CreateSubscription other = (CreateSubscription) obj;
|
||||
return (this.subscriptionId.equals(other.subscriptionId));
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package timely.api;
|
||||
|
||||
public class RemoveSubscription extends SubscriptionRequest {
|
||||
|
||||
private String operation = "remove";
|
||||
private String metric = null;
|
||||
|
||||
public String getOperation() {
|
||||
return operation;
|
||||
}
|
||||
|
||||
public String getMetric() {
|
||||
return metric;
|
||||
}
|
||||
|
||||
public void setMetric(String metric) {
|
||||
this.metric = metric;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return (operation + subscriptionId + metric).hashCode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (null == obj) {
|
||||
return false;
|
||||
}
|
||||
if (obj == this) {
|
||||
return true;
|
||||
}
|
||||
if (obj instanceof RemoveSubscription) {
|
||||
RemoveSubscription other = (RemoveSubscription) obj;
|
||||
return (this.subscriptionId.equals(other.subscriptionId) && this.metric.equals(other.metric));
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package timely.api;
|
||||
|
||||
public abstract class SubscriptionRequest {
|
||||
|
||||
protected String subscriptionId = null;
|
||||
|
||||
public final String getSubscriptionId() {
|
||||
return subscriptionId;
|
||||
}
|
||||
|
||||
public final void setSubscriptionId(String subscriptionId) {
|
||||
this.subscriptionId = subscriptionId;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
package timely.api;
|
||||
|
||||
public class VersionRequest {
|
||||
|
||||
private String operation = "version";
|
||||
|
||||
public String getOperation() {
|
||||
return operation;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package timely.api.response;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.lang3.builder.EqualsBuilder;
|
||||
import org.apache.commons.lang3.builder.HashCodeBuilder;
|
||||
import org.apache.commons.lang3.builder.ToStringBuilder;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
|
||||
|
||||
import timely.model.Metric;
|
||||
import timely.model.Tag;
|
||||
|
||||
@JsonPropertyOrder({"metric", "timestamp", "value", "tags", "subscriptionId", "complete"})
|
||||
public class MetricResponse {
|
||||
|
||||
private String metric;
|
||||
private long timestamp;
|
||||
private double value;
|
||||
private List<Tag> tags;
|
||||
private String subscriptionId;
|
||||
private boolean complete = false;
|
||||
|
||||
public String getMetric() {
|
||||
return metric;
|
||||
}
|
||||
|
||||
public long getTimestamp() {
|
||||
return timestamp;
|
||||
}
|
||||
|
||||
public double getValue() {
|
||||
return value;
|
||||
}
|
||||
|
||||
public List<Tag> getTags() {
|
||||
return tags;
|
||||
}
|
||||
|
||||
public boolean isComplete() {
|
||||
return complete;
|
||||
}
|
||||
|
||||
public void setMetric(String metric) {
|
||||
this.metric = metric;
|
||||
}
|
||||
|
||||
public void setTimestamp(long timestamp) {
|
||||
this.timestamp = timestamp;
|
||||
}
|
||||
|
||||
public void setValue(double value) {
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
public void setTags(List<Tag> tags) {
|
||||
this.tags = tags;
|
||||
}
|
||||
|
||||
public String getSubscriptionId() {
|
||||
return subscriptionId;
|
||||
}
|
||||
|
||||
public void setSubscriptionId(String subscriptionId) {
|
||||
this.subscriptionId = subscriptionId;
|
||||
}
|
||||
|
||||
public void setComplete(boolean closed) {
|
||||
this.complete = closed;
|
||||
}
|
||||
|
||||
public static MetricResponse fromMetric(Metric metric, String subscriptionId) {
|
||||
MetricResponse response = new MetricResponse();
|
||||
response.setMetric(metric.getName());
|
||||
response.setTimestamp(metric.getValue().getTimestamp());
|
||||
response.setTags(metric.getTags());
|
||||
response.setValue(metric.getValue().getMeasure());
|
||||
response.setSubscriptionId(subscriptionId);
|
||||
response.setComplete(false);
|
||||
return response;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
HashCodeBuilder hash = new HashCodeBuilder();
|
||||
hash.append(this.metric);
|
||||
hash.append(this.timestamp);
|
||||
hash.append(this.value);
|
||||
hash.append(this.tags);
|
||||
hash.append(this.subscriptionId);
|
||||
hash.append(this.complete);
|
||||
return hash.hashCode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (null == obj) {
|
||||
return false;
|
||||
}
|
||||
if (this == obj) {
|
||||
return true;
|
||||
}
|
||||
if (!(obj instanceof MetricResponse)) {
|
||||
return false;
|
||||
}
|
||||
MetricResponse other = (MetricResponse) obj;
|
||||
EqualsBuilder equals = new EqualsBuilder();
|
||||
equals.append(this.metric, other.metric);
|
||||
equals.append(this.timestamp, other.timestamp);
|
||||
equals.append(this.value, other.value);
|
||||
equals.append(this.tags, other.tags);
|
||||
equals.append(this.subscriptionId, other.subscriptionId);
|
||||
equals.append(this.complete, other.complete);
|
||||
return equals.isEquals();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
ToStringBuilder tsb = new ToStringBuilder(this);
|
||||
tsb.append("metric", this.metric);
|
||||
tsb.append("timestamp", this.timestamp);
|
||||
tsb.append("value", this.value);
|
||||
tsb.append("tags", this.tags);
|
||||
tsb.append("subscriptionId", this.subscriptionId);
|
||||
tsb.append("complete", this.complete);
|
||||
return tsb.toString();
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package timely.api.response;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class MetricResponses {
|
||||
|
||||
private List<MetricResponse> responses = new ArrayList<>();
|
||||
|
||||
public synchronized List<MetricResponse> getResponses() {
|
||||
return this.responses;
|
||||
}
|
||||
|
||||
public synchronized void setResponses(List<MetricResponse> responses) {
|
||||
this.responses = responses;
|
||||
}
|
||||
|
||||
public synchronized void addResponse(MetricResponse r) {
|
||||
this.responses.add(r);
|
||||
}
|
||||
|
||||
public int size() {
|
||||
return this.responses.size();
|
||||
}
|
||||
|
||||
public synchronized void clear() {
|
||||
this.responses.clear();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package timely.client.http;
|
||||
|
||||
import javax.net.ssl.HostnameVerifier;
|
||||
import javax.net.ssl.SSLContext;
|
||||
|
||||
import org.apache.http.client.CookieStore;
|
||||
import org.apache.http.client.config.CookieSpecs;
|
||||
import org.apache.http.client.config.RequestConfig;
|
||||
import org.apache.http.conn.ssl.NoopHostnameVerifier;
|
||||
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.apache.http.impl.client.HttpClientBuilder;
|
||||
import org.apache.http.impl.client.HttpClients;
|
||||
import org.glassfish.tyrus.client.SslContextConfigurator;
|
||||
|
||||
public class HttpClient {
|
||||
|
||||
public static SSLContext getSSLContext(String trustStoreFile, String trustStoreType, String trustStorePass, String keyStoreFile, String keyStoreType,
|
||||
String keyStorePass) {
|
||||
SslContextConfigurator scc = new SslContextConfigurator();
|
||||
scc.setTrustStoreFile(trustStoreFile);
|
||||
scc.setTrustStoreType(trustStoreType);
|
||||
if (null != trustStorePass) {
|
||||
scc.setTrustStorePassword(trustStorePass);
|
||||
}
|
||||
scc.setKeyStoreFile(keyStoreFile);
|
||||
scc.setKeyStoreType(keyStoreType);
|
||||
if (null != keyStorePass) {
|
||||
scc.setKeyStorePassword(keyStorePass);
|
||||
}
|
||||
return scc.createSSLContext();
|
||||
}
|
||||
|
||||
public static CloseableHttpClient get(String trustStoreFile, String trustStoreType, String trustStorePass, String keyStoreFile, String keyStoreType,
|
||||
String keyStorePass, CookieStore cookieStore, boolean hostVerificationEnabled) {
|
||||
|
||||
SSLContext ssl = getSSLContext(trustStoreFile, trustStoreType, trustStorePass, keyStoreFile, keyStoreType, keyStorePass);
|
||||
return get(ssl, cookieStore, hostVerificationEnabled);
|
||||
}
|
||||
|
||||
public static CloseableHttpClient get(SSLContext ssl, CookieStore cookieStore, boolean hostVerificationEnabled) {
|
||||
return get(ssl, cookieStore, hostVerificationEnabled, false);
|
||||
}
|
||||
|
||||
public static CloseableHttpClient get(SSLContext ssl, CookieStore cookieStore, boolean hostVerificationEnabled, boolean clientAuth) {
|
||||
HostnameVerifier hostnameVerifier;
|
||||
if (hostVerificationEnabled) {
|
||||
hostnameVerifier = SSLConnectionSocketFactory.getDefaultHostnameVerifier();
|
||||
} else {
|
||||
hostnameVerifier = new NoopHostnameVerifier();
|
||||
}
|
||||
RequestConfig defaultRequestConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.STANDARD).build();
|
||||
HttpClientBuilder builder;
|
||||
if (cookieStore == null) {
|
||||
builder = HttpClients.custom().setSSLContext(ssl).disableCookieManagement().setDefaultRequestConfig(defaultRequestConfig)
|
||||
.setSSLHostnameVerifier(hostnameVerifier);
|
||||
} else {
|
||||
builder = HttpClients.custom().setSSLContext(ssl).setDefaultCookieStore(cookieStore).setDefaultRequestConfig(defaultRequestConfig)
|
||||
.setSSLHostnameVerifier(hostnameVerifier);
|
||||
}
|
||||
|
||||
if (clientAuth) {
|
||||
SSLConnectionSocketFactory sslCSF = new SSLConnectionSocketFactory(ssl, new String[] {"TLSv1.1", "TLSv1.2"}, null, hostnameVerifier);
|
||||
builder.setSSLSocketFactory(sslCSF);
|
||||
}
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package timely.client.tcp;
|
||||
|
||||
import java.io.BufferedWriter;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.net.Socket;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class TcpClient implements AutoCloseable {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(TcpClient.class);
|
||||
|
||||
private final String host;
|
||||
private final int port;
|
||||
private Socket sock = null;
|
||||
private BufferedWriter out = null;
|
||||
private long connectTime = 0L;
|
||||
private long backoff = 2000;
|
||||
private int bufferSize = -1;
|
||||
private int writesSinceFlush = 0;
|
||||
|
||||
public TcpClient(String hostname, int port) {
|
||||
this(hostname, port, -1);
|
||||
}
|
||||
|
||||
public TcpClient(String hostname, int port, int bufferSize) {
|
||||
this.host = hostname;
|
||||
this.port = port;
|
||||
this.bufferSize = bufferSize;
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens a TCP connection to the specified host and port
|
||||
*
|
||||
* @throws IOException
|
||||
* if an error occurs
|
||||
*/
|
||||
public void open() throws IOException {
|
||||
if (connect() != 0) {
|
||||
throw new IOException();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a metric to Timely
|
||||
*
|
||||
* @param metric
|
||||
* newline terminated string representation of Timely metric
|
||||
*/
|
||||
public synchronized void write(String metric) throws IOException {
|
||||
if (connect() != 0) {
|
||||
throw new IOException();
|
||||
}
|
||||
out.write(metric);
|
||||
writesSinceFlush++;
|
||||
if (bufferSize > 0 && writesSinceFlush >= bufferSize) {
|
||||
flush();
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized void flush() throws IOException {
|
||||
if (null != out) {
|
||||
out.flush();
|
||||
writesSinceFlush = 0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes the tcp connection to Timely
|
||||
*
|
||||
* @throws IOException
|
||||
* if an error occurs
|
||||
*/
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
log.trace("Shutting down connection to Timely at {}:{}", host, port);
|
||||
if (null != sock) {
|
||||
try {
|
||||
if (null != out) {
|
||||
if (bufferSize > 0) {
|
||||
flush();
|
||||
}
|
||||
out.close();
|
||||
}
|
||||
sock.close();
|
||||
} catch (IOException e) {
|
||||
log.error("Error closing connection to Timely at " + host + ":" + port + ". Error: " + e.getMessage());
|
||||
} finally {
|
||||
sock = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private synchronized int connect() {
|
||||
if (null == sock || !sock.isConnected()) {
|
||||
if (System.currentTimeMillis() > (connectTime + backoff)) {
|
||||
OutputStreamWriter osw = null;
|
||||
try {
|
||||
connectTime = System.currentTimeMillis();
|
||||
sock = new Socket(host, port);
|
||||
osw = new OutputStreamWriter(sock.getOutputStream(), StandardCharsets.UTF_8);
|
||||
out = new BufferedWriter(osw);
|
||||
backoff = 2000;
|
||||
log.trace("Connected to Timely at {}:{}", host, port);
|
||||
} catch (Exception e) {
|
||||
log.error("Error connecting to Timely at {}:{} - {}", host, port, e.getMessage());
|
||||
if (sock != null) {
|
||||
try {
|
||||
sock.close();
|
||||
} catch (IOException e1) {
|
||||
log.error(e1.getMessage());
|
||||
} finally {
|
||||
sock = null;
|
||||
}
|
||||
}
|
||||
if (osw != null) {
|
||||
try {
|
||||
osw.close();
|
||||
} catch (IOException e1) {
|
||||
log.error(e1.getMessage());
|
||||
}
|
||||
}
|
||||
if (out != null) {
|
||||
try {
|
||||
out.close();
|
||||
} catch (IOException e1) {
|
||||
log.error(e1.getMessage());
|
||||
} finally {
|
||||
out = null;
|
||||
}
|
||||
}
|
||||
backoff = backoff * 2;
|
||||
log.info("Will retry connection in {} ms.", backoff);
|
||||
return -1;
|
||||
}
|
||||
} else {
|
||||
log.warn("Not writing to Timely, waiting to reconnect");
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
public long getConnectTime() {
|
||||
return connectTime;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package timely.client.udp;
|
||||
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.DatagramPacket;
|
||||
import java.net.DatagramSocket;
|
||||
import java.net.InetSocketAddress;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class UdpClient implements AutoCloseable {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(UdpClient.class);
|
||||
|
||||
private final String host;
|
||||
private final int port;
|
||||
private final InetSocketAddress address;
|
||||
private final DatagramPacket packet;
|
||||
private DatagramSocket sock;
|
||||
|
||||
public UdpClient(String host, int port) {
|
||||
this.address = new InetSocketAddress(host, port);
|
||||
this.packet = new DatagramPacket("".getBytes(UTF_8), 0, 0, address.getAddress(), port);
|
||||
this.host = host;
|
||||
this.port = port;
|
||||
}
|
||||
|
||||
public void open() throws IOException {
|
||||
if (null == sock) {
|
||||
this.sock = new DatagramSocket();
|
||||
}
|
||||
}
|
||||
|
||||
public void write(String metric) throws IOException {
|
||||
if (null == this.sock) {
|
||||
throw new IllegalStateException("Must call open first");
|
||||
}
|
||||
this.packet.setData(metric.getBytes(UTF_8));
|
||||
log.info("writing '" + metric + "' to " + this.host + ":" + this.port);
|
||||
this.sock.send(packet);
|
||||
}
|
||||
|
||||
public void flush() throws IOException {}
|
||||
|
||||
public void close() throws IOException {
|
||||
if (null != sock) {
|
||||
sock.close();
|
||||
sock = null;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package timely.client.websocket;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import javax.websocket.CloseReason;
|
||||
import javax.websocket.Endpoint;
|
||||
import javax.websocket.EndpointConfig;
|
||||
import javax.websocket.Session;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public abstract class ClientHandler extends Endpoint {
|
||||
|
||||
private final static Logger log = LoggerFactory.getLogger(ClientHandler.class);
|
||||
|
||||
@Override
|
||||
public void onOpen(Session session, EndpointConfig config) {
|
||||
log.info("Websocket session {} opened.", session.getId());
|
||||
session.addMessageHandler(String.class, message -> {
|
||||
log.info("Message received on Websocket session {}: {}", session.getId(), message);
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onClose(Session session, CloseReason reason) {
|
||||
log.info("Websocket session {} closed.", session.getId());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onError(Session session, Throwable error) {
|
||||
log.error("Error occurred on Websocket session" + session.getId(), error);
|
||||
}
|
||||
|
||||
public void beforeRequest(Map<String,List<String>> headers) {}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package timely.client.websocket;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import javax.websocket.ClientEndpointConfig;
|
||||
import javax.websocket.Decoder;
|
||||
import javax.websocket.Encoder;
|
||||
import javax.websocket.Extension;
|
||||
|
||||
import org.apache.http.cookie.Cookie;
|
||||
|
||||
public class TimelyEndpointConfig implements ClientEndpointConfig {
|
||||
|
||||
private String sessionCookie = null;
|
||||
private final ClientHandler clientEndpoint;
|
||||
|
||||
private static final String FORMAT = "%s=%s;";
|
||||
|
||||
public TimelyEndpointConfig(ClientHandler clientEndpoint, Cookie sessionCookie) {
|
||||
if (null != sessionCookie) {
|
||||
this.sessionCookie = String.format(FORMAT, sessionCookie.getName(), sessionCookie.getValue());
|
||||
}
|
||||
this.clientEndpoint = clientEndpoint;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Class<? extends Encoder>> getEncoders() {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Class<? extends Decoder>> getDecoders() {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String,Object> getUserProperties() {
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getPreferredSubprotocols() {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Extension> getExtensions() {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Configurator getConfigurator() {
|
||||
return new Configurator() {
|
||||
|
||||
@Override
|
||||
public void beforeRequest(Map<String,List<String>> headers) {
|
||||
super.beforeRequest(headers);
|
||||
if (null != sessionCookie) {
|
||||
headers.put("Cookie", Collections.singletonList(sessionCookie));
|
||||
}
|
||||
TimelyEndpointConfig.this.clientEndpoint.beforeRequest(headers);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
package timely.client.websocket;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import javax.net.ssl.SSLContext;
|
||||
import javax.websocket.DeploymentException;
|
||||
import javax.websocket.Session;
|
||||
|
||||
import org.apache.http.HttpResponse;
|
||||
import org.apache.http.HttpStatus;
|
||||
import org.apache.http.client.HttpResponseException;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.client.methods.HttpRequestBase;
|
||||
import org.apache.http.client.utils.HttpClientUtils;
|
||||
import org.apache.http.cookie.Cookie;
|
||||
import org.apache.http.impl.client.BasicCookieStore;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.glassfish.tyrus.client.ClientManager;
|
||||
import org.glassfish.tyrus.client.ClientProperties;
|
||||
import org.glassfish.tyrus.client.SslEngineConfigurator;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.google.common.base.Preconditions;
|
||||
|
||||
import timely.client.http.HttpClient;
|
||||
|
||||
public class WebSocketClient implements AutoCloseable {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(WebSocketClient.class);
|
||||
|
||||
private final String timelyHostname;
|
||||
private final int timelyHttpsPort;
|
||||
private final int timelyWssPort;
|
||||
private final boolean doLogin;
|
||||
private final boolean clientAuth;
|
||||
private final boolean hostVerificationEnabled;
|
||||
private final int bufferSize;
|
||||
private final SSLContext ssl;
|
||||
|
||||
private ClientManager webSocketClient = null;
|
||||
protected Session session = null;
|
||||
protected volatile boolean closed = true;
|
||||
|
||||
protected WebSocketClient(SSLContext ssl, String timelyHostname, int timelyHttpsPort, int timelyWssPort, boolean clientAuth, boolean doLogin,
|
||||
boolean hostVerificationEnabled, int bufferSize) {
|
||||
this.ssl = ssl;
|
||||
this.timelyHostname = timelyHostname;
|
||||
this.timelyHttpsPort = timelyHttpsPort;
|
||||
this.timelyWssPort = timelyWssPort;
|
||||
this.doLogin = doLogin;
|
||||
this.clientAuth = clientAuth;
|
||||
this.hostVerificationEnabled = hostVerificationEnabled;
|
||||
this.bufferSize = bufferSize;
|
||||
|
||||
Preconditions.checkNotNull(timelyHostname, "%s must be supplied", "Timely host name");
|
||||
Preconditions.checkNotNull(timelyHttpsPort, "%s must be supplied", "Timely HTTPS port");
|
||||
Preconditions.checkNotNull(timelyWssPort, "%s must be supplied", "Timely WSS port");
|
||||
}
|
||||
|
||||
protected WebSocketClient(String timelyHostname, int timelyHttpsPort, int timelyWssPort, boolean clientAuth, boolean doLogin, String keyStoreFile,
|
||||
String keyStoreType, String keyStorePass, String trustStoreFile, String trustStoreType, String trustStorePass,
|
||||
boolean hostVerificationEnabled, int bufferSize) {
|
||||
this(HttpClient.getSSLContext(trustStoreFile, trustStoreType, trustStorePass, keyStoreFile, keyStoreType, keyStorePass), timelyHostname,
|
||||
timelyHttpsPort, timelyWssPort, clientAuth, doLogin, hostVerificationEnabled, bufferSize);
|
||||
}
|
||||
|
||||
public void open(ClientHandler clientEndpoint) throws IOException, DeploymentException, URISyntaxException {
|
||||
Cookie sessionCookie = null;
|
||||
if (doLogin) {
|
||||
BasicCookieStore cookieJar = new BasicCookieStore();
|
||||
try (CloseableHttpClient client = HttpClient.get(ssl, cookieJar, hostVerificationEnabled, clientAuth)) {
|
||||
|
||||
String target = "https://" + timelyHostname + ":" + timelyHttpsPort + "/login";
|
||||
HttpRequestBase request = null;
|
||||
// HTTP GET to /login to use certificate based login
|
||||
request = new HttpGet(target);
|
||||
log.trace("Performing client certificate login");
|
||||
|
||||
HttpResponse response = null;
|
||||
try {
|
||||
response = client.execute(request);
|
||||
if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
|
||||
throw new HttpResponseException(response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase());
|
||||
}
|
||||
for (Cookie c : cookieJar.getCookies()) {
|
||||
if (c.getName().equals("TSESSIONID")) {
|
||||
sessionCookie = c;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (null == sessionCookie) {
|
||||
throw new IllegalStateException("Unable to find TSESSIONID cookie header in Timely login response");
|
||||
}
|
||||
} finally {
|
||||
HttpClientUtils.closeQuietly(response);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
SslEngineConfigurator sslEngine = new SslEngineConfigurator(ssl);
|
||||
sslEngine.setClientMode(true);
|
||||
sslEngine.setHostVerificationEnabled(hostVerificationEnabled);
|
||||
|
||||
webSocketClient = ClientManager.createClient();
|
||||
webSocketClient.getProperties().put(ClientProperties.SSL_ENGINE_CONFIGURATOR, sslEngine);
|
||||
webSocketClient.getProperties().put(ClientProperties.INCOMING_BUFFER_SIZE, bufferSize);
|
||||
String wssPath = "wss://" + timelyHostname + ":" + timelyWssPort + "/websocket";
|
||||
session = webSocketClient.connectToServer(clientEndpoint, new TimelyEndpointConfig(clientEndpoint, sessionCookie), new URI(wssPath));
|
||||
|
||||
final ByteBuffer pingData = ByteBuffer.allocate(0);
|
||||
webSocketClient.getScheduledExecutorService().scheduleAtFixedRate(() -> {
|
||||
try {
|
||||
session.getBasicRemote().sendPing(pingData);
|
||||
} catch (Exception e) {
|
||||
log.error("Error sending ping", e);
|
||||
}
|
||||
}, 30, 60, TimeUnit.SECONDS);
|
||||
closed = false;
|
||||
}
|
||||
|
||||
public synchronized void close() throws IOException {
|
||||
if (closed) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
if (null != webSocketClient) {
|
||||
webSocketClient.shutdown();
|
||||
}
|
||||
} finally {
|
||||
session = null;
|
||||
webSocketClient = null;
|
||||
closed = true;
|
||||
}
|
||||
}
|
||||
|
||||
public boolean isClosed() {
|
||||
return closed;
|
||||
}
|
||||
|
||||
}
|
||||
+92
@@ -0,0 +1,92 @@
|
||||
package timely.client.websocket.subscription;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URISyntaxException;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.Future;
|
||||
|
||||
import javax.net.ssl.SSLContext;
|
||||
import javax.websocket.CloseReason;
|
||||
import javax.websocket.DeploymentException;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import timely.api.AddSubscription;
|
||||
import timely.api.CloseSubscription;
|
||||
import timely.api.CreateSubscription;
|
||||
import timely.api.RemoveSubscription;
|
||||
import timely.api.VersionRequest;
|
||||
import timely.client.websocket.ClientHandler;
|
||||
import timely.client.websocket.WebSocketClient;
|
||||
import timely.serialize.JsonSerializer;
|
||||
|
||||
public class WebSocketSubscriptionClient extends WebSocketClient {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(WebSocketSubscriptionClient.class);
|
||||
private final String subscriptionId;
|
||||
|
||||
public WebSocketSubscriptionClient(SSLContext ssl, String timelyHostname, int timelyHttpsPort, int timelyWssPort, boolean clientAuth, boolean doLogin,
|
||||
boolean hostVerificationEnabled, int bufferSize) {
|
||||
super(ssl, timelyHostname, timelyHttpsPort, timelyWssPort, clientAuth, doLogin, hostVerificationEnabled, bufferSize);
|
||||
subscriptionId = UUID.randomUUID().toString();
|
||||
log.trace("Created WebSocketClient with subscriptionId {}", this.subscriptionId);
|
||||
}
|
||||
|
||||
public WebSocketSubscriptionClient(String timelyHostname, int timelyHttpsPort, int timelyWssPort, boolean clientAuth, boolean doLogin, String keyStoreFile,
|
||||
String keyStoreType, String keyStorePass, String trustStoreFile, String trustStoreType, String trustStorePass,
|
||||
boolean hostVerificationEnabled, int bufferSize) {
|
||||
super(timelyHostname, timelyHttpsPort, timelyWssPort, clientAuth, doLogin, keyStoreFile, keyStoreType, keyStorePass, trustStoreFile, trustStoreType,
|
||||
trustStorePass, hostVerificationEnabled, bufferSize);
|
||||
subscriptionId = UUID.randomUUID().toString();
|
||||
log.trace("Created WebSocketClient with subscriptionId {}", this.subscriptionId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void open(ClientHandler clientEndpoint) throws IOException, DeploymentException, URISyntaxException {
|
||||
super.open(clientEndpoint);
|
||||
CreateSubscription create = new CreateSubscription();
|
||||
create.setSubscriptionId(subscriptionId);
|
||||
session.getBasicRemote().sendText(JsonSerializer.getObjectMapper().writeValueAsString(create));
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void close() throws IOException {
|
||||
if (null != session) {
|
||||
CloseSubscription close = new CloseSubscription();
|
||||
close.setSubscriptionId(subscriptionId);
|
||||
try {
|
||||
session.getBasicRemote().sendText(JsonSerializer.getObjectMapper().writeValueAsString(close));
|
||||
session.close(new CloseReason(CloseReason.CloseCodes.NORMAL_CLOSURE, "Client called close."));
|
||||
} catch (Exception e) {
|
||||
log.info("Unable to send close message to server: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
super.close();
|
||||
}
|
||||
|
||||
public Future<Void> addSubscription(String metric, Map<String,String> tags, long startTime, long endTime, long delayTime) throws IOException {
|
||||
AddSubscription add = new AddSubscription();
|
||||
add.setSubscriptionId(subscriptionId);
|
||||
add.setMetric(metric);
|
||||
add.setTags(Optional.ofNullable(tags));
|
||||
add.setStartTime(Optional.ofNullable(startTime));
|
||||
add.setEndTime(Optional.ofNullable(endTime));
|
||||
add.setDelayTime(Optional.ofNullable(delayTime));
|
||||
return session.getAsyncRemote().sendText(JsonSerializer.getObjectMapper().writeValueAsString(add));
|
||||
}
|
||||
|
||||
public Future<Void> removeSubscription(String metric) throws Exception {
|
||||
RemoveSubscription remove = new RemoveSubscription();
|
||||
remove.setSubscriptionId(subscriptionId);
|
||||
remove.setMetric(metric);
|
||||
return session.getAsyncRemote().sendText(JsonSerializer.getObjectMapper().writeValueAsString(remove));
|
||||
}
|
||||
|
||||
public Future<Void> requestVersion() throws Exception {
|
||||
VersionRequest version = new VersionRequest();
|
||||
return session.getAsyncRemote().sendText(JsonSerializer.getObjectMapper().writeValueAsString(version));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package timely.serialize;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonParser;
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
|
||||
|
||||
public class JsonSerializer {
|
||||
|
||||
private static ObjectMapper mapper = new ObjectMapper();
|
||||
|
||||
static {
|
||||
mapper.registerModule(new Jdk8Module());
|
||||
mapper.configure(DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true);
|
||||
mapper.configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, true);
|
||||
mapper.configure(JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS, true);
|
||||
}
|
||||
|
||||
public static ObjectMapper getObjectMapper() {
|
||||
return mapper;
|
||||
}
|
||||
}
|
||||
Loaded 100 of 592 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user