Skip to content

Commit 01dbfee

Browse files
author
twitter-team
committed
Open-sourcing Tweetypie
Tweetypie is the core Tweet service that handles the reading and writing of Tweet data.
1 parent 90d7ea3 commit 01dbfee

File tree

591 files changed

+68352
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

591 files changed

+68352
-0
lines changed

Diff for: README.md

+1
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ Product surfaces at Twitter are built on a shared set of data, models, and softw
2424
| | [timelines-aggregation-framework](timelines/data_processing/ml_util/aggregation_framework/README.md) | Framework for generating aggregate features in batch or real time. |
2525
| | [representation-manager](representation-manager/README.md) | Service to retrieve embeddings (i.e. SimClusers and TwHIN). |
2626
| | [twml](twml/README.md) | Legacy machine learning framework built on TensorFlow v1. |
27+
| | [Tweetypie](tweetypie/server/README.md) | Core Tweet service that handles the reading and writing of Tweet data. |
2728

2829
The product surface currently included in this repository is the For You Timeline.
2930

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
package com.twitter.tweetypie.additionalfields
2+
3+
import com.twitter.tweetypie.thriftscala.Tweet
4+
import com.twitter.scrooge.TFieldBlob
5+
import com.twitter.scrooge.ThriftStructField
6+
7+
object AdditionalFields {
8+
type FieldId = Short
9+
10+
/** additional fields really start at 100, be we are ignoring conversation id for now */
11+
val StartAdditionalId = 101
12+
13+
/** all known [[Tweet]] field IDs */
14+
val CompiledFieldIds: Seq[FieldId] = Tweet.metaData.fields.map(_.id)
15+
16+
/** all known [[Tweet]] fields in the "additional-field" range (excludes id) */
17+
val CompiledAdditionalFieldMetaDatas: Seq[ThriftStructField[Tweet]] =
18+
Tweet.metaData.fields.filter(f => isAdditionalFieldId(f.id))
19+
20+
val CompiledAdditionalFieldsMap: Map[Short, ThriftStructField[Tweet]] =
21+
CompiledAdditionalFieldMetaDatas.map(field => (field.id, field)).toMap
22+
23+
/** all known [[Tweet]] field IDs in the "additional-field" range */
24+
val CompiledAdditionalFieldIds: Seq[FieldId] =
25+
CompiledAdditionalFieldsMap.keys.toSeq
26+
27+
/** all [[Tweet]] field IDs which should be rejected when set as additional
28+
* fields on via PostTweetRequest.additionalFields or RetweetRequest.additionalFields */
29+
val RejectedFieldIds: Seq[FieldId] = Seq(
30+
// Should be provided via PostTweetRequest.conversationControl field. go/convocontrolsbackend
31+
Tweet.ConversationControlField.id,
32+
// This field should only be set based on whether the client sets the right community
33+
// tweet annotation.
34+
Tweet.CommunitiesField.id,
35+
// This field should not be set by clients and should opt for
36+
// [[PostTweetRequest.ExclusiveTweetControlOptions]].
37+
// The exclusiveTweetControl field requires the userId to be set
38+
// and we shouldn't trust the client to provide the right one.
39+
Tweet.ExclusiveTweetControlField.id,
40+
// This field should not be set by clients and should opt for
41+
// [[PostTweetRequest.TrustedFriendsControlOptions]].
42+
// The trustedFriendsControl field requires the trustedFriendsListId to be
43+
// set and we shouldn't trust the client to provide the right one.
44+
Tweet.TrustedFriendsControlField.id,
45+
// This field should not be set by clients and should opt for
46+
// [[PostTweetRequest.CollabControlOptions]].
47+
// The collabControl field requires a list of Collaborators to be
48+
// set and we shouldn't trust the client to provide the right one.
49+
Tweet.CollabControlField.id
50+
)
51+
52+
def isAdditionalFieldId(fieldId: FieldId): Boolean =
53+
fieldId >= StartAdditionalId
54+
55+
/**
56+
* Provides a list of all additional field IDs on the tweet, which include all
57+
* the compiled additional fields and all the provided passthrough fields. This includes
58+
* compiled additional fields where the value is None.
59+
*/
60+
def allAdditionalFieldIds(tweet: Tweet): Seq[FieldId] =
61+
CompiledAdditionalFieldIds ++ tweet._passthroughFields.keys
62+
63+
/**
64+
* Provides a list of all field IDs that have a value on the tweet which are not known compiled
65+
* additional fields (excludes [[Tweet.id]]).
66+
*/
67+
def unsettableAdditionalFieldIds(tweet: Tweet): Seq[FieldId] =
68+
CompiledFieldIds
69+
.filter { id =>
70+
!isAdditionalFieldId(id) && id != Tweet.IdField.id && tweet.getFieldBlob(id).isDefined
71+
} ++
72+
tweet._passthroughFields.keys
73+
74+
/**
75+
* Provides a list of all field IDs that have a value on the tweet which are explicitly disallowed
76+
* from being set via PostTweetRequest.additionalFields and RetweetRequest.additionalFields
77+
*/
78+
def rejectedAdditionalFieldIds(tweet: Tweet): Seq[FieldId] =
79+
RejectedFieldIds
80+
.filter { id => tweet.getFieldBlob(id).isDefined }
81+
82+
def unsettableAdditionalFieldIdsErrorMessage(unsettableFieldIds: Seq[FieldId]): String =
83+
s"request may not contain fields: [${unsettableFieldIds.sorted.mkString(", ")}]"
84+
85+
/**
86+
* Provides a list of all additional field IDs that have a value on the tweet,
87+
* compiled and passthrough (excludes Tweet.id).
88+
*/
89+
def nonEmptyAdditionalFieldIds(tweet: Tweet): Seq[FieldId] =
90+
CompiledAdditionalFieldMetaDatas.collect {
91+
case f if f.getValue(tweet) != None => f.id
92+
} ++ tweet._passthroughFields.keys
93+
94+
def additionalFields(tweet: Tweet): Seq[TFieldBlob] =
95+
(tweet.getFieldBlobs(CompiledAdditionalFieldIds) ++ tweet._passthroughFields).values.toSeq
96+
97+
/**
98+
* Merge base tweet with additional fields.
99+
* Non-additional fields in the additional tweet are ignored.
100+
* @param base: a tweet that contains basic fields
101+
* @param additional: a tweet object that carries additional fields
102+
*/
103+
def setAdditionalFields(base: Tweet, additional: Tweet): Tweet =
104+
setAdditionalFields(base, additionalFields(additional))
105+
106+
def setAdditionalFields(base: Tweet, additional: Option[Tweet]): Tweet =
107+
additional.map(setAdditionalFields(base, _)).getOrElse(base)
108+
109+
def setAdditionalFields(base: Tweet, additional: Traversable[TFieldBlob]): Tweet =
110+
additional.foldLeft(base) { case (t, f) => t.setField(f) }
111+
112+
/**
113+
* Unsets the specified fields on the given tweet.
114+
*/
115+
def unsetFields(tweet: Tweet, fieldIds: Iterable[FieldId]): Tweet = {
116+
tweet.unsetFields(fieldIds.toSet)
117+
}
118+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
scala_library(
2+
sources = ["*.scala"],
3+
compiler_option_sets = ["fatal_warnings"],
4+
platform = "java8",
5+
strict_deps = True,
6+
tags = ["bazel-compatible"],
7+
dependencies = [
8+
"3rdparty/jvm/org/apache/thrift:libthrift",
9+
"mediaservices/commons/src/main/thrift:thrift-scala",
10+
"scrooge/scrooge-core",
11+
"src/thrift/com/twitter/escherbird:media-annotation-structs-scala",
12+
"src/thrift/com/twitter/spam/rtf:safety-label-scala",
13+
"tweetypie/common/src/thrift/com/twitter/tweetypie:tweet-scala",
14+
],
15+
)
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
scala_library(
2+
compiler_option_sets = ["fatal_warnings"],
3+
strict_deps = True,
4+
tags = ["bazel-compatible"],
5+
dependencies = [
6+
"finagle/finagle-memcached/src/main/scala",
7+
"scrooge/scrooge-serializer",
8+
"stitch/stitch-core",
9+
"util/util-core",
10+
"util/util-logging",
11+
# CachedValue struct
12+
"tweetypie/servo/repo/src/main/thrift:thrift-scala",
13+
"util/util-slf4j-api/src/main/scala/com/twitter/util/logging",
14+
],
15+
)

0 commit comments

Comments
 (0)