A team of researchers at USC’s Information Sciences Institute, collaborating with the CAll for Regulation support In Social MediA, is analyzing harmful content — misinformation, hate speech, spam, bots, trolls and more — on social media platforms to create a body of work that will inform social media companies, regulators and fellow academics how content moderation can be improved.

CARISMA staff hope to use this body of knowledge to create a content moderation simulator. This simulator will allow researchers to play around with different techniques and methods for content moderation, ranging from temporarily and permanently suspending users to flagging, demoting and removing content.

Daily headlines, sent straight to your inbox.

Subscribe to our newsletter to keep up with the latest at and around USC.

(function(global) {
function serialize(form){if(!form||form.nodeName!==”FORM”){return }var i,j,q=[];for(i=form.elements.length-1;i>=0;i=i-1){if(form.elements[i].name===””){continue}switch(form.elements[i].nodeName){case”INPUT”:switch(form.elements[i].type){case”text”:case”hidden”:case”password”:case”button”:case”reset”:case”submit”:q.push(form.elements[i].name+”=”+encodeURIComponent(form.elements[i].value));break;case”checkbox”:case”radio”:if(form.elements[i].checked){q.push(form.elements[i].name+”=”+encodeURIComponent(form.elements[i].value))}break;case”file”:break}break;case”TEXTAREA”:q.push(form.elements[i].name+”=”+encodeURIComponent(form.elements[i].value));break;case”SELECT”:switch(form.elements[i].type){case”select-one”:q.push(form.elements[i].name+”=”+encodeURIComponent(form.elements[i].value));break;case”select-multiple”:for(j=form.elements[i].options.length-1;j>=0;j=j-1){if(form.elements[i].options[j].selected){q.push(form.elements[i].name+”=”+encodeURIComponent(form.elements[i].options[j].value))}}break}break;case”BUTTON”:switch(form.elements[i].type){case”reset”:case”submit”:case”button”:q.push(form.elements[i].name+”=”+encodeURIComponent(form.elements[i].value));break}break}}return q.join(“&”)};

function extend(destination, source) {
for (var prop in source) {
destination[prop] = source[prop];
}
}

if (!Mimi) var Mimi = {};
if (!Mimi.Signups) Mimi.Signups = {};

Mimi.Signups.EmbedValidation = function() {
this.initialize();

var _this = this;
if (document.addEventListener) {
this.form.addEventListener(‘submit’, function(e){
_this.onFormSubmit(e);
});
} else {
this.form.attachEvent(‘onsubmit’, function(e){
_this.onFormSubmit(e);
});
}
};

extend(Mimi.Signups.EmbedValidation.prototype, {
initialize: function() {
this.form = document.getElementById(’ema_signup_form’);
this.submit = document.getElementById(‘webform_submit_button’);
this.callbackName = ‘jsonp_callback_’ + Math.round(100000 * Math.random());
this.validEmail = /.+@.+..+/
},

onFormSubmit: function(e) {
e.preventDefault();

this.validate();
if (this.isValid) {
this.submitForm();
} else {
this.revalidateOnChange();
}
},

validate: function() {
this.isValid = true;
this.emailValidation();
this.fieldAndListValidation();
this.updateFormAfterValidation();
},

emailValidation: function() {
var email = document.getElementById(‘signup_email’);

if (this.validEmail.test(email.value)) {
this.removeTextFieldError(email);
} else {
this.textFieldError(email);
this.isValid = false;
}
},

fieldAndListValidation: function() {
var fields = this.form.querySelectorAll(‘.mimi_field.required’);

for (var i = 0; i = 0) {
return ‘checkboxes’;
} else {
return ‘text_field’;
}
},

checkboxAndRadioValidation: function(field) {
var inputs = field.getElementsByTagName(‘input’),
selected = false;

for (var i = 0; i < inputs.length; ++i) {
var input = inputs[i];
if((input.type === 'checkbox' || input.type === 'radio') && input.checked) {
selected = true;
}
}

if (selected) {
field.className = field.className.replace(/ invalid/g, '');
} else {
if (field.className.indexOf('invalid') === -1) {
field.className += ' invalid';
}

this.isValid = false;
}
},

textAndDropdownValidation: function(field, type) {
var inputs = field.getElementsByTagName('input');

for (var i = 0; i = 0) {
if (type === ‘text_field’) {
this.textValidation(input);
} else {
this.dropdownValidation(field, input);
}
}
}
this.htmlEmbedDropdownValidation(field);
},

textValidation: function(input) {
if (input.id === ‘signup_email’) return;

if (input.value) {
this.removeTextFieldError(input);
} else {
this.textFieldError(input);
this.isValid = false;
}
},

dropdownValidation: function(field, input) {
if (input.value) {
field.className = field.className.replace(/ invalid/g, ”);
} else {
if (field.className.indexOf(‘invalid’) === -1) field.className += ‘ invalid’;
this.onSelectCallback(input);
this.isValid = false;
}
},

htmlEmbedDropdownValidation: function(field) {
var dropdowns = field.querySelectorAll(‘.mimi_html_dropdown’);
var _this = this;

for (var i = 0; i < dropdowns.length; ++i) {
var dropdown = dropdowns[i];

if (dropdown.value) {
field.className = field.className.replace(/ invalid/g, '');
} else {
if (field.className.indexOf('invalid') === -1) field.className += ' invalid';
this.isValid = false;
dropdown.onchange = (function(){ _this.validate(); });
}
}
},

textFieldError: function(input) {
input.className = 'required invalid';
input.placeholder = input.getAttribute('data-required-field');
},

removeTextFieldError: function(input) {
input.className = 'required';
input.placeholder = '';
},

onSelectCallback: function(input) {
if (typeof Widget === 'undefined' || !Widget.BasicDropdown) return;

var dropdownEl = input.parentNode,
instances = Widget.BasicDropdown.instances,
_this = this;

for (var i = 0; i < instances.length; ++i) {
var instance = instances[i];
if (instance.wrapperEl === dropdownEl) {
instance.onSelect = function(){ _this.validate() };
}
}
},

updateFormAfterValidation: function() {
this.form.className = this.setFormClassName();
this.submit.value = this.submitButtonText();
this.submit.disabled = !this.isValid;
this.submit.className = this.isValid ? 'submit' : 'disabled';
},

setFormClassName: function() {
var name = this.form.className;

if (this.isValid) {
return name.replace(/s?mimi_invalid/, '');
} else {
if (name.indexOf('mimi_invalid') === -1) {
return name += ' mimi_invalid';
} else {
return name;
}
}
},

submitButtonText: function() {
var invalidFields = document.querySelectorAll('.invalid'),
text;

if (this.isValid || !invalidFields) {
text = this.submit.getAttribute('data-default-text');
} else {
if (invalidFields.length || invalidFields[0].className.indexOf('checkgroup') === -1) {
text = this.submit.getAttribute('data-invalid-text');
} else {
text = this.submit.getAttribute('data-choose-list');
}
}
return text;
},

submitForm: function() {
this.formSubmitting();

var _this = this;
window[this.callbackName] = function(response) {
delete window[this.callbackName];
document.body.removeChild(script);
_this.onSubmitCallback(response);
};

var script = document.createElement('script');
script.src = this.formUrl('json');
document.body.appendChild(script);
},

formUrl: function(format) {
var action = this.form.action;
if (format === 'json') action += '.json';
return action + '?callback=' + this.callbackName + '&' + serialize(this.form);
},

formSubmitting: function() {
this.form.className += ' mimi_submitting';
this.submit.value = this.submit.getAttribute('data-submitting-text');
this.submit.disabled = true;
this.submit.className = 'disabled';
},

onSubmitCallback: function(response) {
if (response.success) {
this.onSubmitSuccess(response.result);
} else {
top.location.href = this.formUrl('html');
}
},

onSubmitSuccess: function(result) {
if (result.has_redirect) {
top.location.href = result.redirect;
} else if(result.single_opt_in || !result.confirmation_html) {
this.disableForm();
this.updateSubmitButtonText(this.submit.getAttribute('data-thanks'));
} else {
this.showConfirmationText(result.confirmation_html);
}
},

showConfirmationText: function(html) {
var fields = this.form.querySelectorAll('.mimi_field');

for (var i = 0; i < fields.length; ++i) {
fields[i].style['display'] = 'none';
}

(this.form.querySelectorAll('fieldset')[0] || this.form).innerHTML = html;
},

disableForm: function() {
var elements = this.form.elements;
for (var i = 0; i < elements.length; ++i) {
elements[i].disabled = true;
}
},

updateSubmitButtonText: function(text) {
this.submit.value = text;
},

revalidateOnChange: function() {
var fields = this.form.querySelectorAll(".mimi_field.required"),
_this = this;

var onTextFieldChange = function() {
if (this.getAttribute('name') === 'signup[email]') {
if (_this.validEmail.test(this.value)) _this.validate();
} else {
if (this.value.length === 1) _this.validate();
}
}

for (var i = 0; i < fields.length; ++i) {
var inputs = fields[i].getElementsByTagName('input');
for (var j = 0; j < inputs.length; ++j) {
if (this.fieldType(fields[i]) === 'text_field') {
inputs[j].onkeyup = onTextFieldChange;
inputs[j].onchange = onTextFieldChange;
} else {
inputs[j].onchange = function(){ _this.validate() };
}
}
}
}
});

if (document.addEventListener) {
document.addEventListener("DOMContentLoaded", function() {
new Mimi.Signups.EmbedValidation();
});
}
else {
window.attachEvent('onload', function() {
new Mimi.Signups.EmbedValidation();
});
}
})(this);

Luca Luceri, a co-author of two research papers under the ISI and CARISMA, is also working on the simulator. 

“We are building this big simulator with … several thousands of agents that try to replicate humans and human’s interaction,” Luceri said. “We want to see what happens if we limit the interaction of good legitimate users with these bad actors.” 

Lucia Perfetti, a freshman majoring in cinema and media studies, estimates she sees harmful content on social media roughly twice a week and she believes that hate speech has had a resurgence. 

“I’ll see lots of hate speech against specifically Jewish communities right now with the conflicts going on abroad,” Perfetti said. “At the [LGBTQ+ Student Center], we were at a meeting and we were looking through a Pride Month post and there were some hate speech comments and we reported them to administration.”

Together, the ISI and CARISMA have published several papers, one of which analyzed how X, formerly known as Twitter, responded to major geopolitical events. The study found that the X content moderation team is more proactive at suspending recently created accounts than older accounts and that suspended accounts engage and post more than other users. 

In a second joint study, researchers analyzed how moderated content — content that a social media company eventually takes down for violating community guidelines — spreads across different platforms. According to the study, harmful content on YouTube gets more engagement than non-moderated content from every other mainstream platforms.  

Luceri recalled being surprised by some of his findings. 

“We didn’t expect to find that moderated videos were getting so much traction and were shared far more than content from other social media platforms,” Luceri said. “These videos got traction quite soon in their lifespan, so that they could reach a wide audience before moderation.”

Angela Wang, a senior majoring in film and television production, said she most often sees hateful comments directed towards celebrities on Instagram. 

“Usually I don’t really comment on stuff like that because I feel like, ‘Why bother?’, because it’s other people’s dumb comments,’” Wang said. “But if something really pisses me off I’ll report it … [Controversial content] gets the views, they grab people’s attention, and people are gonna want to comment on it because they’re like, ‘This is so wrong.’”

Luceri’s team also found that during the 2020 United States presidential election, people who supported former President Trump were more likely to share YouTube videos on X that were eventually taken down than those who supported other candidates.  

Perfetti said she noticed a lot of pushback against Trump and his followers’ statements during the 2020 presidential election.

“[Trump’s] followers had a larger presence and thus there was probably a [higher] likelihood of their posts getting taken down,” Perfetti said. “There’s a lot of pressure on the company, which I think is great, like [the] public having a say in what companies regulate. But that can also be really damaging if companies get more biased against one side or the other.”

The type of research that Luceri does is largely possible through using tools known as application programming interfaces to collect and compile necessary data — such as millions of tweets. However, the lack of information available through YouTube’s and X’s API prevented Luceri’s team from expanding their research.  

Sometimes, a social media platform’s API only tells researchers half the story. 

“When an account [is] suspended on [X], we know only whether it was suspended or not, but we couldn’t get any information on when the account was suspended,” Luceri. “So we can speculate and try to go back in time and see what account was this user, but we have no proof, no signal or evidence from the API.” 

Luceri said that he’s also mindful of the balance at play between one’s freedom of speech and the need for content moderation. 

 “If we moderate a hateful comment or a hateful post, this goes against the freedom of speech,” Luceri said. “But on the other side, we need also to take into account that these spaces require some moderation. So if we allow everybody [to speak] in a certain way — violent, toxic or hateful … We needed a way to somehow polish these spaces.”

One of CARISMA’s objectives is to learn which interventions are most effective at mitigating harms like misinformation and hate speech. 

“As a researcher, I think I’m driven by this mission to make these spaces healthier and safer,” Luceri said.