Skip to content

Commit 94ff4ca

Browse files
author
twitter-team
committed
Open-sourcing Topic Social Proof Service
Topic Social Proof Service (TSPS) delivers highly relevant topics tailored to a user's interests by analyzing topic preferences, such as following or unfollowing, and employing semantic annotations and other machine learning models.
1 parent 138bb51 commit 94ff4ca

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

51 files changed

+3489
-0
lines changed

‎topic-social-proof/README.md

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
# Topic Social Proof Service (TSPS)
2+
=================
3+
4+
**Topic Social Proof Service** (TSPS) serves as a centralized source for verifying topics related to Timelines and Notifications. By analyzing user's topic preferences, such as following or unfollowing, and employing semantic annotations and tweet embeddings from SimClusters, or other machine learning models, TSPS delivers highly relevant topics tailored to each user's interests.
5+
6+
For instance, when a tweet discusses Stephen Curry, the service determines if the content falls under topics like "NBA" and/or "Golden State Warriors" while also providing relevance scores based on SimClusters Embedding. Additionally, TSPS evaluates user-specific topic preferences to offer a comprehensive list of available topics, only those the user is currently following, or new topics they have not followed but may find interesting if recommended on specific product surfaces.
7+
8+

‎topic-social-proof/server/BUILD

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
jvm_binary(
2+
name = "bin",
3+
basename = "topic-social-proof",
4+
main = "com.twitter.tsp.TopicSocialProofStratoFedServerMain",
5+
runtime_platform = "java11",
6+
tags = [
7+
"bazel-compatible",
8+
],
9+
dependencies = [
10+
"strato/src/main/scala/com/twitter/strato/logging/logback",
11+
"topic-social-proof/server/src/main/resources",
12+
"topic-social-proof/server/src/main/scala/com/twitter/tsp",
13+
],
14+
)
15+
16+
# Aurora Workflows build phase convention requires a jvm_app named with ${project-name}-app
17+
jvm_app(
18+
name = "topic-social-proof-app",
19+
archive = "zip",
20+
binary = ":bin",
21+
tags = [
22+
"bazel-compatible",
23+
],
24+
)

‎topic-social-proof/server/src/main/resources/BUILD

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
resources(
2+
sources = [
3+
"*.xml",
4+
"*.yml",
5+
"config/*.yml",
6+
],
7+
tags = ["bazel-compatible"],
8+
)

‎topic-social-proof/server/src/main/resources/config/decider.yml

+61
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
# Keys are sorted in an alphabetical order
2+
3+
enable_topic_social_proof_score:
4+
comment : "Enable the calculation of <topic, tweet> cosine similarity score in TopicSocialProofStore. 0 means do not calculate the score and use a random rank to generate topic social proof"
5+
default_availability: 0
6+
7+
enable_tweet_health_score:
8+
comment: "Enable the calculation for health scores in tweetInfo. By enabling this decider, we will compute TweetHealthModelScore"
9+
default_availability: 0
10+
11+
enable_user_agatha_score:
12+
comment: "Enable the calculation for health scores in tweetInfo. By enabling this decider, we will compute UserHealthModelScore"
13+
default_availability: 0
14+
15+
enable_loadshedding_HomeTimeline:
16+
comment: "Enable loadshedding (from 0% to 100%). Requests that have been shed will return an empty response"
17+
default_availability: 0
18+
19+
enable_loadshedding_HomeTimelineTopicTweets:
20+
comment: "Enable loadshedding (from 0% to 100%). Requests that have been shed will return an empty response"
21+
default_availability: 0
22+
23+
enable_loadshedding_HomeTimelineRecommendTopicTweets:
24+
comment: "Enable loadshedding (from 0% to 100%). Requests that have been shed will return an empty response"
25+
default_availability: 0
26+
27+
enable_loadshedding_MagicRecsRecommendTopicTweets:
28+
comment: "Enable loadshedding (from 0% to 100%). Requests that have been shed will return an empty response"
29+
default_availability: 0
30+
31+
enable_loadshedding_TopicLandingPage:
32+
comment: "Enable loadshedding (from 0% to 100%). Requests that have been shed will return an empty response"
33+
default_availability: 0
34+
35+
enable_loadshedding_HomeTimelineFeatures:
36+
comment: "Enable loadshedding (from 0% to 100%). Requests that have been shed will return an empty response"
37+
default_availability: 0
38+
39+
enable_loadshedding_HomeTimelineTopicTweetsMetrics:
40+
comment: "Enable loadshedding (from 0% to 100%). Requests that have been shed will return an empty response"
41+
default_availability: 0
42+
43+
enable_loadshedding_HomeTimelineUTEGTopicTweets:
44+
comment: "Enable loadshedding (from 0% to 100%). Requests that have been shed will return an empty response"
45+
default_availability: 0
46+
47+
enable_loadshedding_HomeTimelineSimClusters:
48+
comment: "Enable loadshedding (from 0% to 100%). Requests that have been shed will return an empty response"
49+
default_availability: 0
50+
51+
enable_loadshedding_ExploreTopicTweets:
52+
comment: "Enable loadshedding (from 0% to 100%). Requests that have been shed will return an empty response"
53+
default_availability: 0
54+
55+
enable_loadshedding_MagicRecsTopicTweets:
56+
comment: "Enable loadshedding (from 0% to 100%). Requests that have been shed will return an empty response"
57+
default_availability: 0
58+
59+
enable_loadshedding_Search:
60+
comment: "Enable loadshedding (from 0% to 100%). Requests that have been shed will return an empty response"
61+
default_availability: 0

‎topic-social-proof/server/src/main/resources/logback.xml

+155
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
<configuration>
2+
<shutdownHook class="ch.qos.logback.core.hook.DelayingShutdownHook"/>
3+
<property name="async_queue_size" value="${queue.size:-50000}"/>
4+
<property name="async_max_flush_time" value="${max.flush.time:-0}"/>
5+
<!-- ===================================================== -->
6+
<!-- Structured Logging -->
7+
<!-- ===================================================== -->
8+
<!-- Only sample 0.1% of the requests -->
9+
<property name="splunk_sampling_rate" value="${splunk_sampling_rate:-0.001}"/>
10+
<include resource="structured-logger-logback.xml"/>
11+
<!-- ===================================================== -->
12+
<!-- Service Config -->
13+
<!-- ===================================================== -->
14+
<property name="DEFAULT_SERVICE_PATTERN"
15+
value="%-16X{transactionId} %logger %msg"/>
16+
17+
<!-- ===================================================== -->
18+
<!-- Common Config -->
19+
<!-- ===================================================== -->
20+
21+
<!-- JUL/JDK14 to Logback bridge -->
22+
<contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator">
23+
<resetJUL>true</resetJUL>
24+
</contextListener>
25+
26+
<!-- Service Log (Rollover every 50MB, max 11 logs) -->
27+
<appender name="SERVICE" class="ch.qos.logback.core.rolling.RollingFileAppender">
28+
<file>${log.service.output}</file>
29+
<rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
30+
<fileNamePattern>${log.service.output}.%i</fileNamePattern>
31+
<minIndex>1</minIndex>
32+
<maxIndex>10</maxIndex>
33+
</rollingPolicy>
34+
<triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
35+
<maxFileSize>50MB</maxFileSize>
36+
</triggeringPolicy>
37+
<encoder>
38+
<pattern>%date %.-3level ${DEFAULT_SERVICE_PATTERN}%n</pattern>
39+
</encoder>
40+
</appender>
41+
42+
<!-- Strato package only log (Rollover every 50MB, max 11 logs) -->
43+
<appender name="STRATO-ONLY" class="ch.qos.logback.core.rolling.RollingFileAppender">
44+
<file>${log.strato_only.output}</file>
45+
<rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
46+
<fileNamePattern>${log.strato_only.output}.%i</fileNamePattern>
47+
<minIndex>1</minIndex>
48+
<maxIndex>10</maxIndex>
49+
</rollingPolicy>
50+
<triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
51+
<maxFileSize>50MB</maxFileSize>
52+
</triggeringPolicy>
53+
<encoder>
54+
<pattern>%date %.-3level ${DEFAULT_SERVICE_PATTERN}%n</pattern>
55+
</encoder>
56+
</appender>
57+
58+
<!-- LogLens -->
59+
<appender name="LOGLENS" class="com.twitter.loglens.logback.LoglensAppender">
60+
<mdcAdditionalContext>true</mdcAdditionalContext>
61+
<category>loglens</category>
62+
<index>${log.lens.index}</index>
63+
<tag>${log.lens.tag}/service</tag>
64+
<encoder>
65+
<pattern>%msg%n</pattern>
66+
</encoder>
67+
<turboFilter class="ch.qos.logback.classic.turbo.DuplicateMessageFilter">
68+
<cacheSize>500</cacheSize>
69+
<allowedRepetitions>50</allowedRepetitions>
70+
</turboFilter>
71+
<filter class="com.twitter.strato.logging.logback.RegexFilter">
72+
<forLogger>manhattan-client</forLogger>
73+
<excludeRegex>.*InvalidRequest.*</excludeRegex>
74+
</filter>
75+
</appender>
76+
77+
<!-- ===================================================== -->
78+
<!-- Primary Async Appenders -->
79+
<!-- ===================================================== -->
80+
81+
<appender name="ASYNC-SERVICE" class="ch.qos.logback.classic.AsyncAppender">
82+
<queueSize>${async_queue_size}</queueSize>
83+
<maxFlushTime>${async_max_flush_time}</maxFlushTime>
84+
<appender-ref ref="SERVICE"/>
85+
</appender>
86+
87+
<appender name="ASYNC-STRATO-ONLY" class="ch.qos.logback.classic.AsyncAppender">
88+
<queueSize>${async_queue_size}</queueSize>
89+
<maxFlushTime>${async_max_flush_time}</maxFlushTime>
90+
<appender-ref ref="STRATO-ONLY"/>
91+
</appender>
92+
93+
<appender name="ASYNC-LOGLENS" class="ch.qos.logback.classic.AsyncAppender">
94+
<queueSize>${async_queue_size}</queueSize>
95+
<maxFlushTime>${async_max_flush_time}</maxFlushTime>
96+
<appender-ref ref="LOGLENS"/>
97+
</appender>
98+
99+
<!-- ===================================================== -->
100+
<!-- Package Config -->
101+
<!-- ===================================================== -->
102+
103+
<!-- Per-Package Config (shared) -->
104+
<logger name="com.twitter" level="info"/>
105+
106+
<!--
107+
By default, we leave the strato package at INFO level.
108+
However, this line allows us to set the entire strato package, or a subset of it, to
109+
a specific level. For example, if you pass -Dstrato_log_package=streaming -Dstrato_log_level=DEBUG
110+
only loggers under com.twitter.strato.streaming.* will be set to DEBUG level. Passing only
111+
-Dstrato_log_level will set all of strato.* to the specified level.
112+
-->
113+
<logger name="com.twitter.strato${strato_log_package:-}" level="${strato_log_level:-INFO}"/>
114+
115+
<logger name="com.twitter.wilyns" level="warn"/>
116+
<logger name="com.twitter.finagle.mux" level="warn"/>
117+
<logger name="com.twitter.finagle.serverset2" level="warn"/>
118+
<logger name="com.twitter.logging.ScribeHandler" level="warn"/>
119+
<logger name="com.twitter.zookeeper.client.internal" level="warn"/>
120+
<logger name="com.twitter.decider.StoreDecider" level="warn"/>
121+
122+
<!-- Per-Package Config (Strato) -->
123+
<logger name="com.twitter.distributedlog.client" level="warn"/>
124+
<logger name="com.twitter.finagle.mtls.authorization.config.AccessControlListConfiguration" level="warn"/>
125+
<logger name="com.twitter.finatra.kafka.common.kerberoshelpers" level="warn"/>
126+
<logger name="com.twitter.finatra.kafka.utils.BootstrapServerUtils" level="warn"/>
127+
<logger name="com.twitter.server.coordinate" level="error"/>
128+
<logger name="com.twitter.zookeeper.client" level="info"/>
129+
<logger name="org.apache.zookeeper" level="error"/>
130+
<logger name="org.apache.zookeeper.ClientCnxn" level="warn"/>
131+
<logger name="ZkSession" level="info"/>
132+
<logger name="OptimisticLockingCache" level="off"/>
133+
<logger name="manhattan-client" level="warn"/>
134+
<logger name="strato.op" level="warn"/>
135+
<logger name="org.apache.kafka.clients.NetworkClient" level="error"/>
136+
<logger name="org.apache.kafka.clients.consumer.internals" level="error"/>
137+
<logger name="org.apache.kafka.clients.producer.internals" level="error"/>
138+
<!-- produce a lot of messages like: Building client authenticator with server name kafka -->
139+
<logger name="org.apache.kafka.common.network" level="warn"/>
140+
141+
<!-- Root Config -->
142+
<root level="${log_level:-INFO}">
143+
<appender-ref ref="ASYNC-SERVICE"/>
144+
<appender-ref ref="ASYNC-LOGLENS"/>
145+
</root>
146+
147+
<!-- Strato package only logging-->
148+
<logger name="com.twitter.strato"
149+
level="info"
150+
additivity="true">
151+
<appender-ref ref="ASYNC-STRATO-ONLY" />
152+
</logger>
153+
154+
155+
</configuration>

‎topic-social-proof/server/src/main/scala/com/twitter/tsp/BUILD

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
scala_library(
2+
compiler_option_sets = ["fatal_warnings"],
3+
tags = [
4+
"bazel-compatible",
5+
],
6+
dependencies = [
7+
"finatra/inject/inject-thrift-client",
8+
"strato/src/main/scala/com/twitter/strato/fed",
9+
"strato/src/main/scala/com/twitter/strato/fed/server",
10+
"topic-social-proof/server/src/main/scala/com/twitter/tsp/columns",
11+
],
12+
)

‎topic-social-proof/server/src/main/scala/com/twitter/tsp/TopicSocialProofStratoFedServer.scala

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
package com.twitter.tsp
2+
3+
import com.google.inject.Module
4+
import com.twitter.strato.fed._
5+
import com.twitter.strato.fed.server._
6+
import com.twitter.strato.warmup.Warmer
7+
import com.twitter.tsp.columns.TopicSocialProofColumn
8+
import com.twitter.tsp.columns.TopicSocialProofBatchColumn
9+
import com.twitter.tsp.handlers.UttChildrenWarmupHandler
10+
import com.twitter.tsp.modules.RepresentationScorerStoreModule
11+
import com.twitter.tsp.modules.GizmoduckUserModule
12+
import com.twitter.tsp.modules.TSPClientIdModule
13+
import com.twitter.tsp.modules.TopicListingModule
14+
import com.twitter.tsp.modules.TopicSocialProofStoreModule
15+
import com.twitter.tsp.modules.TopicTweetCosineSimilarityAggregateStoreModule
16+
import com.twitter.tsp.modules.TweetInfoStoreModule
17+
import com.twitter.tsp.modules.TweetyPieClientModule
18+
import com.twitter.tsp.modules.UttClientModule
19+
import com.twitter.tsp.modules.UttLocalizationModule
20+
import com.twitter.util.Future
21+
22+
object TopicSocialProofStratoFedServerMain extends TopicSocialProofStratoFedServer
23+
24+
trait TopicSocialProofStratoFedServer extends StratoFedServer {
25+
override def dest: String = "/s/topic-social-proof/topic-social-proof"
26+
27+
override val modules: Seq[Module] =
28+
Seq(
29+
GizmoduckUserModule,
30+
RepresentationScorerStoreModule,
31+
TopicSocialProofStoreModule,
32+
TopicListingModule,
33+
TopicTweetCosineSimilarityAggregateStoreModule,
34+
TSPClientIdModule,
35+
TweetInfoStoreModule,
36+
TweetyPieClientModule,
37+
UttClientModule,
38+
UttLocalizationModule
39+
)
40+
41+
override def columns: Seq[Class[_ <: StratoFed.Column]] =
42+
Seq(
43+
classOf[TopicSocialProofColumn],
44+
classOf[TopicSocialProofBatchColumn]
45+
)
46+
47+
override def configureWarmer(warmer: Warmer): Unit = {
48+
warmer.add(
49+
"uttChildrenWarmupHandler",
50+
() => {
51+
handle[UttChildrenWarmupHandler]()
52+
Future.Unit
53+
}
54+
)
55+
}
56+
}

‎topic-social-proof/server/src/main/scala/com/twitter/tsp/columns/BUILD

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
scala_library(
2+
compiler_option_sets = ["fatal_warnings"],
3+
tags = [
4+
"bazel-compatible",
5+
],
6+
dependencies = [
7+
"stitch/stitch-storehaus",
8+
"strato/src/main/scala/com/twitter/strato/fed",
9+
"topic-social-proof/server/src/main/scala/com/twitter/tsp/service",
10+
"topic-social-proof/server/src/main/thrift:thrift-scala",
11+
],
12+
)

0 commit comments

Comments
 (0)