<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD JATS (Z39.96) Journal Publishing DTD v1.1d3 20150301//EN" "http://jats.nlm.nih.gov/publishing/1.1d3/JATS-journalpublishing1.dtd">
<article article-type="research-article" dtd-version="1.1d3" xml:lang="en" xmlns:mml="http://www.w3.org/1998/Math/MathML" xmlns:xlink="http://www.w3.org/1999/xlink">
<front>
<journal-meta>
<journal-id journal-id-type="nlm-ta">PLoS ONE</journal-id>
<journal-id journal-id-type="publisher-id">plos</journal-id>
<journal-id journal-id-type="pmc">plosone</journal-id>
<journal-title-group>
<journal-title>PLOS ONE</journal-title>
</journal-title-group>
<issn pub-type="epub">1932-6203</issn>
<publisher>
<publisher-name>Public Library of Science</publisher-name>
<publisher-loc>San Francisco, CA USA</publisher-loc>
</publisher>
</journal-meta>
<article-meta>
<article-id pub-id-type="doi">10.1371/journal.pone.0307814</article-id>
<article-id pub-id-type="publisher-id">PONE-D-23-25545</article-id>
<article-categories>
<subj-group subj-group-type="heading">
<subject>Research Article</subject>
</subj-group>
<subj-group subj-group-type="Discipline-v3">
<subject>Research and analysis methods</subject><subj-group><subject>Research assessment</subject></subj-group></subj-group><subj-group subj-group-type="Discipline-v3">
<subject>Science policy</subject><subj-group><subject>Research integrity</subject></subj-group></subj-group><subj-group subj-group-type="Discipline-v3">
<subject>Medicine and health sciences</subject></subj-group><subj-group subj-group-type="Discipline-v3">
<subject>Science policy</subject><subj-group><subject>Science and technology workforce</subject><subj-group><subject>Careers in research</subject><subj-group><subject>Scientists</subject></subj-group></subj-group></subj-group></subj-group><subj-group subj-group-type="Discipline-v3">
<subject>People and places</subject><subj-group><subject>Population groupings</subject><subj-group><subject>Professions</subject><subj-group><subject>Scientists</subject></subj-group></subj-group></subj-group></subj-group><subj-group subj-group-type="Discipline-v3">
<subject>Research and analysis methods</subject><subj-group><subject>Research assessment</subject><subj-group><subject>Research quality assessment</subject></subj-group></subj-group></subj-group><subj-group subj-group-type="Discipline-v3">
<subject>Biology and life sciences</subject><subj-group><subject>Psychology</subject><subj-group><subject>Personality</subject></subj-group></subj-group></subj-group><subj-group subj-group-type="Discipline-v3">
<subject>Social sciences</subject><subj-group><subject>Psychology</subject><subj-group><subject>Personality</subject></subj-group></subj-group></subj-group><subj-group subj-group-type="Discipline-v3">
<subject>Social sciences</subject><subj-group><subject>Economics</subject><subj-group><subject>Labor economics</subject><subj-group><subject>Employment</subject><subj-group><subject>Careers</subject></subj-group></subj-group></subj-group></subj-group></subj-group><subj-group subj-group-type="Discipline-v3">
<subject>Biology and life sciences</subject><subj-group><subject>Psychology</subject><subj-group><subject>Psychometrics</subject><subj-group><subject>Personality tests</subject></subj-group></subj-group></subj-group></subj-group><subj-group subj-group-type="Discipline-v3">
<subject>Social sciences</subject><subj-group><subject>Psychology</subject><subj-group><subject>Psychometrics</subject><subj-group><subject>Personality tests</subject></subj-group></subj-group></subj-group></subj-group></article-categories>
<title-group>
<article-title>Relational responsibilities: Researchers perspective on current and progressive assessment criteria: A focus group study</article-title>
<alt-title alt-title-type="running-head">Relational responsibilities: Researchers perspective on current and progressive assessment criteria</alt-title>
</title-group>
<contrib-group>
<contrib contrib-type="author" corresp="yes" xlink:type="simple">
<contrib-id authenticated="true" contrib-id-type="orcid">https://orcid.org/0000-0002-1826-2274</contrib-id>
<name name-style="western">
<surname>Tijdink</surname>
<given-names>Joeri K.</given-names>
</name>
<role content-type="http://credit.niso.org/contributor-roles/conceptualization/">Conceptualization</role>
<role content-type="http://credit.niso.org/contributor-roles/data-curation/">Data curation</role>
<role content-type="http://credit.niso.org/contributor-roles/formal-analysis/">Formal analysis</role>
<role content-type="http://credit.niso.org/contributor-roles/funding-acquisition/">Funding acquisition</role>
<role content-type="http://credit.niso.org/contributor-roles/investigation/">Investigation</role>
<role content-type="http://credit.niso.org/contributor-roles/methodology/">Methodology</role>
<role content-type="http://credit.niso.org/contributor-roles/software/">Software</role>
<role content-type="http://credit.niso.org/contributor-roles/validation/">Validation</role>
<role content-type="http://credit.niso.org/contributor-roles/visualization/">Visualization</role>
<role content-type="http://credit.niso.org/contributor-roles/writing-original-draft/">Writing – original draft</role>
<role content-type="http://credit.niso.org/contributor-roles/writing-review-editing/">Writing – review &amp; editing</role>
<xref ref-type="aff" rid="aff001"><sup>1</sup></xref>
<xref ref-type="aff" rid="aff002"><sup>2</sup></xref>
<xref ref-type="aff" rid="aff003"><sup>3</sup></xref>
<xref ref-type="corresp" rid="cor001">*</xref>
</contrib>
<contrib contrib-type="author" xlink:type="simple">
<name name-style="western">
<surname>Valkenburg</surname>
<given-names>Govert</given-names>
</name>
<role content-type="http://credit.niso.org/contributor-roles/conceptualization/">Conceptualization</role>
<role content-type="http://credit.niso.org/contributor-roles/formal-analysis/">Formal analysis</role>
<role content-type="http://credit.niso.org/contributor-roles/funding-acquisition/">Funding acquisition</role>
<role content-type="http://credit.niso.org/contributor-roles/methodology/">Methodology</role>
<role content-type="http://credit.niso.org/contributor-roles/supervision/">Supervision</role>
<role content-type="http://credit.niso.org/contributor-roles/writing-review-editing/">Writing – review &amp; editing</role>
<xref ref-type="aff" rid="aff004"><sup>4</sup></xref>
</contrib>
<contrib contrib-type="author" xlink:type="simple">
<name name-style="western">
<surname>Rijcke</surname>
<given-names>Sarah de</given-names>
</name>
<role content-type="http://credit.niso.org/contributor-roles/conceptualization/">Conceptualization</role>
<role content-type="http://credit.niso.org/contributor-roles/funding-acquisition/">Funding acquisition</role>
<role content-type="http://credit.niso.org/contributor-roles/investigation/">Investigation</role>
<role content-type="http://credit.niso.org/contributor-roles/project-administration/">Project administration</role>
<role content-type="http://credit.niso.org/contributor-roles/supervision/">Supervision</role>
<role content-type="http://credit.niso.org/contributor-roles/writing-review-editing/">Writing – review &amp; editing</role>
<xref ref-type="aff" rid="aff005"><sup>5</sup></xref>
</contrib>
<contrib contrib-type="author" xlink:type="simple">
<name name-style="western">
<surname>Dix</surname>
<given-names>Guus</given-names>
</name>
<role content-type="http://credit.niso.org/contributor-roles/formal-analysis/">Formal analysis</role>
<role content-type="http://credit.niso.org/contributor-roles/investigation/">Investigation</role>
<role content-type="http://credit.niso.org/contributor-roles/methodology/">Methodology</role>
<role content-type="http://credit.niso.org/contributor-roles/supervision/">Supervision</role>
<role content-type="http://credit.niso.org/contributor-roles/validation/">Validation</role>
<role content-type="http://credit.niso.org/contributor-roles/visualization/">Visualization</role>
<role content-type="http://credit.niso.org/contributor-roles/writing-original-draft/">Writing – original draft</role>
<role content-type="http://credit.niso.org/contributor-roles/writing-review-editing/">Writing – review &amp; editing</role>
<xref ref-type="aff" rid="aff006"><sup>6</sup></xref>
</contrib>
</contrib-group>
<aff id="aff001"><label>1</label> <addr-line>AmsterdamUMC, Department of Ethics, Health and Humanities, Amsterdam, the Netherlands</addr-line></aff>
<aff id="aff002"><label>2</label> <addr-line>Department of Philosophy, VU University, Amsterdam, the Netherlands</addr-line></aff>
<aff id="aff003"><label>3</label> <addr-line>Faculty of Psychology, Universitas Padjadjaram, Bandung, Indonesia</addr-line></aff>
<aff id="aff004"><label>4</label> <addr-line>Department of Interdisciplinary Studies of Culture, Norwegian University of Science and Technology, Trondheim, Norway</addr-line></aff>
<aff id="aff005"><label>5</label> <addr-line>Center for Science and Technology Studies (CWTS), Leiden University, Leiden, the Netherlands</addr-line></aff>
<aff id="aff006"><label>6</label> <addr-line>Faculty of Behavioural, Management and Social Sciences (BMS), Universiteit Twente, Enschede, the Netherlands</addr-line></aff>
<contrib-group>
<contrib contrib-type="editor" xlink:type="simple">
<name name-style="western">
<surname>Cortes</surname>
<given-names>Julian D.</given-names>
</name>
<role>Editor</role>
<xref ref-type="aff" rid="edit1"/>
</contrib>
</contrib-group>
<aff id="edit1"><addr-line>Universidad del Rosario: Universidad Del Rosario, COLOMBIA</addr-line></aff>
<author-notes>
<fn fn-type="conflict" id="coi001">
<p>The authors have declared that no competing interests exist.</p>
</fn>
<corresp id="cor001">* E-mail: <email xlink:type="simple">j.tijdink@amsterdamumc.nl</email></corresp>
</author-notes>
<pub-date pub-type="epub">
<day>4</day>
<month>9</month>
<year>2024</year>
</pub-date>
<pub-date pub-type="collection">
<year>2024</year>
</pub-date>
<volume>19</volume>
<issue>9</issue>
<elocation-id>e0307814</elocation-id>
<history>
<date date-type="received">
<day>30</day>
<month>8</month>
<year>2023</year>
</date>
<date date-type="accepted">
<day>11</day>
<month>7</month>
<year>2024</year>
</date>
</history>
<permissions>
<copyright-year>2024</copyright-year>
<copyright-holder>Tijdink et al</copyright-holder>
<license xlink:href="http://creativecommons.org/licenses/by/4.0/" xlink:type="simple">
<license-p>This is an open access article distributed under the terms of the <ext-link ext-link-type="uri" xlink:href="http://creativecommons.org/licenses/by/4.0/" xlink:type="simple">Creative Commons Attribution License</ext-link>, which permits unrestricted use, distribution, and reproduction in any medium, provided the original author and source are credited.</license-p>
</license>
</permissions>
<self-uri content-type="pdf" xlink:href="info:doi/10.1371/journal.pone.0307814"/>
<abstract>
<sec id="sec001">
<title>Introduction</title>
<p>The focus on quantitative indicators–number of publications and grants, journal impact factors, Hirsch-index–has become pervasive in research management, funding systems, and research and publication practices (SES). Accountability through performance measurement has become the gold standard to increase productivity and (cost-) efficiency in academia. Scientific careers are strongly shaped by the push to produce more in a veritable ‘publish or perish‘ culture. To this end, we investigated the perspectives of biomedical researchers on responsible assessment criteria that foster responsible conduct of research.</p>
</sec>
<sec id="sec002">
<title>Methods</title>
<p>We performed a qualitative focus group study among 3 University medical centers in the Netherlands. In these centers, we performed 2 randomly selected groups of early career researchers (PhD and postdoc level &amp; senior researchers (associate and full professors) from these 3 institutions and explored how relational responsibilities relate to responsible conduct of research and inquired how potential (formal) assessment criteria could correspond with these responsibilities.</p>
</sec>
<sec id="sec003">
<title>Results</title>
<p>In this study we highlighted what is considered responsible research among junior and senior researchers in the Netherlands and how this can be assessed in formal assessment criteria. The participants reflected on responsible research and highlighted several academic responsibilities (such as supervision, collaboration and teaching) that are often overlooked and that are considered a crucial prerequisite for responsible research. As these responsibilities pertain to intercollegiate relations, we henceforth refer to them as relational. After our systematic analysis of these relational responsibilities, participants suggested some ideas to improve current assessment criteria. We focused on how these duties can be reflected in multidimensional, concrete and sustainable assessment criteria. Focus group participants emphasized the importance of assessing team science (both individual as collective), suggested the use of a narrative in researcher assessment and valued the use of 360 degrees assessment of researchers. Participants believed that these alternative assessments, centered on relational responsibilities, could help in fostering responsible research practices. However, participants stressed that unclarity about the new assessment criteria would only cause more publication stress and insecurity about evaluation of their performance.</p>
</sec>
<sec id="sec004">
<title>Conclusion</title>
<p>Our study suggests that relational responsibilities should ideally play a more prominent role in future assessment criteria as they correspond with and aspire the practice of responsible research. Our participants gave several suggestions how to make these skills quantifiable and assessable in future assessment criteria. However, the development of these criteria is still in its infancy, implementation can cause uncertainties among those assessed and consequently, future research should focus on how to make these criteria more tangible, concrete and applicable in daily practice to make them applicable to measure and assess responsible research practices in institutions.</p>
</sec>
<sec id="sec005">
<title>Trial registration</title>
<p>Open Science Framework <ext-link ext-link-type="uri" xlink:href="https://osf.io/9tjda/" xlink:type="simple">https://osf.io/9tjda/</ext-link>.</p>
</sec>
</abstract>
<funding-group>
<award-group id="award001">
<funding-source>
<institution-wrap>
<institution-id institution-id-type="funder-id">http://dx.doi.org/10.13039/501100001826</institution-id>
<institution>ZonMw</institution>
</institution-wrap>
</funding-source>
<award-id>Grant No. 445001010</award-id>
<principal-award-recipient>
<name name-style="western">
<surname>Rijcke</surname>
<given-names>Sarah de</given-names>
</name>
</principal-award-recipient>
</award-group>
<funding-statement>ZonMw (= funder in the Netherlands) Award Number: Grant No. 445001010 | Recipient: Sarah de Rijcke, Prof. PhD The funders had no role in study design, data collection and analysis, decision to publish, or preparation of the manuscript.</funding-statement>
</funding-group>
<counts>
<fig-count count="0"/>
<table-count count="2"/>
<page-count count="17"/>
</counts>
<custom-meta-group>
<custom-meta id="data-availability">
<meta-name>Data Availability</meta-name>
<meta-value>All relevant data are within the manuscript and its <xref ref-type="sec" rid="sec030">Supporting Information</xref> files.</meta-value>
</custom-meta>
</custom-meta-group>
</article-meta>
</front>
<body>
<sec id="sec006" sec-type="intro">
<title>Introduction</title>
<p>The focus on quantitative indicators–number of publications and grants, journal impact factors, Hirsch-index–has become pervasive in research management, funding systems, and research and publication practices (SES). Accountability through performance measurement has become the gold standard to increase productivity and (cost-) efficiency in academia. Scientific careers are strongly shaped by the push to produce more in a veritable ‘publish or perish‘ culture [<xref ref-type="bibr" rid="pone.0307814.ref001">1</xref>, <xref ref-type="bibr" rid="pone.0307814.ref002">2</xref>]. Over the past decade, the shortcomings of assessment criteria for promotion and tenure for researchers have become increasingly evident [<xref ref-type="bibr" rid="pone.0307814.ref003">3</xref>–<xref ref-type="bibr" rid="pone.0307814.ref006">6</xref>]. Such criteria may stimulate undesirable research practices such as guest/ghost authorship, multi-publishing in predatory journals, salami slicing and p-hacking with the aim to increase performance metrics. These criteria jeopardize the responsible conduct of research. Besides, it may induce fierce competition, false incentives, reduced willingness to collaborate and reduced attention to other important academic duties (e.g. education). Moreover, other research also emphasized reform of evaluation practices to stimulate flexibility per individual assessment, institutions should incentivize non-traditional criteria [<xref ref-type="bibr" rid="pone.0307814.ref007">7</xref>] and emphasizing interdisciplinary work [<xref ref-type="bibr" rid="pone.0307814.ref008">8</xref>], would bring new perspectives to the debate on promotion and tenure. There have been a number of initiatives recently that aim to change current assessment practices (the Leiden Manifesto [<xref ref-type="bibr" rid="pone.0307814.ref006">6</xref>], the DORA statement [<xref ref-type="bibr" rid="pone.0307814.ref009">9</xref>], the changing incentives and reward criteria in funding bodies [<xref ref-type="bibr" rid="pone.0307814.ref010">10</xref>, <xref ref-type="bibr" rid="pone.0307814.ref011">11</xref>], the Reward alliance and the Hong Kong manifesto [<xref ref-type="bibr" rid="pone.0307814.ref012">12</xref>].</p>
<p>However, two major unknowns still stand out.</p>
<p>First, it is unclear to what extent institutions will actually change their evaluation practices. A recent study, investigating such practices in 146 institutions, shows that, despite the criticism, many institutions still favor traditional assessment criteria over novel and innovative criteria that emphasize compliance with open science practices, good leadership, responsible supervision or educational skills [<xref ref-type="bibr" rid="pone.0307814.ref007">7</xref>]. Regarding the reliability of scientific knowledge, similarly, universities might remain rather conservative in not including the transparency, replicability and accuracy of reported findings in new-found assessment criteria [<xref ref-type="bibr" rid="pone.0307814.ref013">13</xref>].</p>
<p>Second, it is yet unclear whether the existing reform initiatives will counter unintended effects of the conventional use of performance metrics. Being relatively new, it is an open question whether or not they help to foster responsible conduct of research by selecting the scientists with more multidimensional profiles beyond a good publication, citation and funding record. Moreover, it is uncertain if these mechanisms will foster academic behaviour that complies with ideals of responsible conduct in research.</p>
<p>In light of these two unknowns, it is important to understand the kinds of bottom-up support that institutional changes in academic assessment can count on. For as a survey of publication practices showed, researchers do (re)consider such practices in relation to institutional evaluation criteria [<xref ref-type="bibr" rid="pone.0307814.ref014">14</xref>]. The dissemination and acceptability of new evaluation practices could benefit from insights into researchers’ perspectives on the current assessment of responsible conduct of research and on the changes in the assessment of academic tasks and responsibilities that are needed to assess research and researchers responsibly. One study alluded to this by emphasizing researchers perceptions of valuation regimes beyond traditional research activities and how current systems and criteria can influence behavior and choices regarding the activities they prioritize [<xref ref-type="bibr" rid="pone.0307814.ref015">15</xref>]. This is also influenced by intense competition can narrow the range of activities and goals that scientists pursue due to the limited set of criteria and values by which scientists are judged and that reevaluating the criteria in academia is necessary to create a healthier and more sustainable research culture [<xref ref-type="bibr" rid="pone.0307814.ref016">16</xref>].</p>
<p>In a two-year qualitative research project, <italic>Optimizing the responsible researcher</italic>, we asked: What possibilities do biomedical researchers themselves see for the improvement of evaluation practices? To systematically address that question, we draw on 6 focus group interviews with junior and senior researchers at three Dutch University Medical Centers (UMC’s).</p>
</sec>
<sec id="sec007" sec-type="materials|methods">
<title>Methods</title>
<sec id="sec008">
<title>Ethical approval</title>
<p>All procedures performed in this study involving human participants were in accordance with the ethical standards of the institutional and/or national research committee. The Medical Ethics Review Committee of VU University Medical Center assessed our project and confirmed that the Medical Research Involving Human Subjects Act (WMO) does not apply to our study (see Supplementary files).</p>
</sec>
<sec id="sec009">
<title>Participants</title>
<p>We included researchers who worked at 3 different University Medical Centers in the Netherlands. We decided to conduct focusgroups among 2 different groups and selected 2 types of active researchers; 1) Senior researchers: assistant, associate and full professors and 2) Junior researchers: postdocs, PhD candidates. We selected those researchers who have published about their research in the past two years, as a proxy criterion for their being ‘active’ as researchers. The two categories are considered sufficiently internally homogeneous with respect to academic seniority, and should allow for the focus group participants to recognize each other as peers. Our recruitment strategy was diverse; we used several techniques to invite colleagues from different ranks and disciplines. We approached the dean’s office to ask for interested researchers, used our collegial network and randomly invited researchers by email to invite them to participate in our study.</p>
<p>We recruited a variety of biomedical disciplines to create some perspectival diversity. Within each group, all participants were selected from the same UMC. We aimed for a group size between 6 and 8 participants, and maintained a minimum of 3 researchers. After confirmation of participation, the participants were sent a one-page description of the purposes of the research project as well as the main conditions of the informed consent in advance of the project by email.</p>
</sec>
<sec id="sec010">
<title>Informed consent</title>
<p>Participation to the focus groups were subject to obtaining verbal informed consent at the start of the focus group, referring to the sent document that contained the privacy policy and based on additional verbal explanation during the focus groups of the following conditions: 1) the purpose of the research and the focus group; 2) treatment of the focus group recordings: anonymized transcriptions and notes on nonverbal communication and behaviour, coding of transcriptions by the research team. And 3) the right to leave the focus group at any point, or to refuse answering specific questions or discussing specific topics, without further obligation of explanation.</p>
</sec>
<sec id="sec011">
<title>Procedure of focus groups</title>
<p>First, we investigated how researchers perceive the norms they see their work subject to, and how they see these norms relate to their own ideals of proper research / responsible research practices. Underlying the focus group methodology is the idea that people come to more balanced and comprehensive account in a conversation with peers, than they would in an individual conversation with an interviewer who is less akin. As set out above, the groups were composed homogeneously with respect to academic rank so participants could recognize each other as peers. Heterogeneity in biomedical disciplines was pursued, as this is reckoned conducive to critical appraisal.</p>
<p>The focus groups were conducted in 2017 and 2018. The focus groups were in English if there were participants that were not fluent in Dutch. A moderator facilitated the focus groups (JKT) and an observer (GV and/or GD) made notes about the process and its content. The focus groups started with a short introduction in which the goals of the project were explained. Questions could be answered in that phase. We used a topic guide as a semi-structured way to guide the focus group discussions.</p>
<p>The focus groups were recorded by the research team and transcribed by an external party. Most transcriptions were coded by 2 members of the research team (JKT and GD). In first instance, code systems were developed and via inductive content analysis led to several themes, and in second instance the themes were analysed by the second coder. After the focus groups, participants received a short summary of the discussion (member check) [<xref ref-type="bibr" rid="pone.0307814.ref017">17</xref>] or received a preprint of this manuscript.</p>
</sec>
<sec id="sec012">
<title>Analysis</title>
<p>We used inductive content analysis for the analysis of the transcripts of the focus groups and took the following steps in this process. First we coded the data on a phrase level. Next, we established a tentative set of themes to analyze the selected phrases, while allowing for flexibility to revise and update these themes during the process, if necessary. Then we analyzed the different themes and summarized these themes. This process of inductive content analysis helps us to understand complex discussions and bring them back to meaningful themes [<xref ref-type="bibr" rid="pone.0307814.ref018">18</xref>]. One researcher (JKT) analyzed and coded the transcripts entirely. One other team member (GD) read the transcripts entirely and coded 2 transcripts independently. The themes were discussed with the other team members [<xref ref-type="bibr" rid="pone.0307814.ref017">17</xref>].</p>
<sec id="sec013">
<title>Descriptive information</title>
<p>We conducted 6 focus groups with 32 researchers from 3 university medical centers and across a wide variety of disciplinary fields, see <xref ref-type="table" rid="pone.0307814.t001">Table 1</xref>.</p>
<table-wrap id="pone.0307814.t001" position="float">
<object-id pub-id-type="doi">10.1371/journal.pone.0307814.t001</object-id>
<label>Table 1</label> <caption><title>Demographic information of focus groups participants.</title></caption>
<alternatives>
<graphic id="pone.0307814.t001g" mimetype="image" position="float" xlink:href="info:doi/10.1371/journal.pone.0307814.t001" xlink:type="simple"/>
<table>
<colgroup>
<col align="left" valign="middle"/>
<col align="left" valign="middle"/>
<col align="left" valign="middle"/>
</colgroup>
<thead>
<tr>
<th align="center">Academic rank</th>
<th align="center">Junior Researchers</th>
<th align="center">Senior researchers</th>
</tr>
</thead>
<tbody>
<tr>
<td align="center"><bold>UMC</bold></td>
<td align="center"/>
<td align="center"/>
</tr>
<tr>
<td align="center">UMC 1</td>
<td align="center">4<sup>/3</sup></td>
<td align="center">4<sup>/3</sup></td>
</tr>
<tr>
<td align="center">UMC 2</td>
<td align="center">3<sup>/1</sup></td>
<td align="center">7<sup>/4</sup></td>
</tr>
<tr>
<td align="center">UMC 3</td>
<td align="center">6 <sup>E/5</sup></td>
<td align="center">8 <sup>E/4</sup></td>
</tr>
</tbody>
</table>
</alternatives>
<table-wrap-foot>
<fn id="t001fn001"><p><sup>E</sup> = focus group was conducted in English; all other focus groups were conducted in Dutch.</p></fn>
<fn id="t001fn002"><p><sup>/x</sup> = number of female participants. UMC = University Medical Center</p></fn>
</table-wrap-foot>
</table-wrap>
</sec>
</sec>
</sec>
<sec id="sec014" sec-type="results">
<title>Results</title>
<p>We started the focus groups with an exercise to familiarize the participants with each other, the topic and the methodology. Participants were asked about their experiences with questionable research practices and what they find the most detrimental research misbehaviors. In both junior and senior groups, most participants had experiences with authorship disputes, while some of them reported more severe breaches of RI (such as data manipulation). Interestingly, bad supervision was mostly mentioned by the more senior participants as a potential threat of research integrity.</p>
<p>In analyzing the content of the focus group transcripts, we came to identify 3 themes–collaboration, supervision and teaching–that relate to responsible research practices. Each theme represents a responsibility of researchers vis-à-vis others that the participants think should be better recognized and rewarded. After this, we explored potential solutions for improving the assessment criteria that they are also in line with the identified themes of the responsible research practices.</p>
<p>Based on these 3 themes that were discussed, we divided them into 2 categories: 1) responsibilities of researchers that are related to research integrity; and 2) new forms of assessment that could cater to these responsibilities. Both categories hence consisted of 3 themes each. We have collected quotes from all focus groups and have reported the most typical quotes that illustrate the discussed solutions in <xref ref-type="table" rid="pone.0307814.t002">Table 2</xref> at the end of this results section. If the focus group was conducted in Dutch, we have translated the quote into English. Below, we present the two categories of themes. After these results of the focus groups, we will briefly discuss the results of the evaluation labs with relevant stakeholders that we organized to safeguard the uptake of our findings.</p>
<table-wrap id="pone.0307814.t002" position="float">
<object-id pub-id-type="doi">10.1371/journal.pone.0307814.t002</object-id>
<label>Table 2</label> <caption><title>Overview of topics and quotes from the focusgroups conducted in our study.</title> <p>Below we highlight the 3 main themes (collaboration, supervision and teaching by emphasizing the most suitable quotes about these themes. Furthermore, we describe the 3 solutions that were brought up during the discussions on researcher assessment in the FGs and highlight potential limitations.</p></caption>
<alternatives>
<graphic id="pone.0307814.t002g" mimetype="image" position="float" xlink:href="info:doi/10.1371/journal.pone.0307814.t002" xlink:type="simple"/>
<table>
<colgroup>
<col align="left" valign="middle"/>
<col align="left" valign="middle"/>
</colgroup>
<tbody>
<tr>
<td align="center" colspan="2"><bold>Responsibilities of researchers that are related to research integrity</bold></td>
</tr>
<tr>
<td align="center" colspan="2"><bold><italic>Theme 1</italic>: <italic>Collaboration should be recognized in alternative assessments</italic></bold></td>
</tr>
<tr>
<td align="left" colspan="2">Senior Researcher UMC 1:<break/><italic>And integrity</italic>: <italic>I cannot see yet how one should assess this</italic>. <italic>It is very important</italic>, <italic>of course</italic>, <italic>for a responsible scientist</italic>, <italic>to have some of these measures that you should ask for in a more qualitative manner or that should be part of the way people assess one another</italic></td>
</tr>
<tr>
<td align="left" colspan="2">Senior Researcher UMC 2:<break/><italic>The only thing that comes to my mind that would be an unbiased</italic>, <italic>but really tricky</italic>, <italic>way to sort of assess honesty</italic>, <italic>is to look at how someone’s work is followed up and reproduced independently</italic>, <italic>right</italic>, <italic>after publication</italic></td>
</tr>
<tr>
<td align="left" colspan="2">Junior researcher UMC 3:<break/><italic>‘being responsible for your researchers is also part of responsible research’</italic></td>
</tr>
<tr>
<td align="left" colspan="2">Junior researcher UMC 2:<break/><italic>‘people who enjoy the game</italic>, <italic>but that are not doing responsible research</italic>, <italic>they stick around too much’</italic></td>
</tr>
<tr>
<td align="left" colspan="2">Junior researcher UMC 1:<break/><italic>‘When you look at the system as it is right now</italic>, <italic>it presorts quite significantly on a very specific</italic>, <italic>scary personality type’</italic></td>
</tr>
<tr>
<td align="left" colspan="2">Senior researcher UMC 3: <break/><italic>‘collaboration</italic>, <italic>the competence to collaborate</italic>, <italic>has to be assessed’</italic></td>
</tr>
<tr>
<td align="left" colspan="2">senior researcher UMC 3:<break/><italic>‘You could also have like</italic>, <italic>well</italic>, <italic>a prize for a well-functioning team; you could have something like a team player prize’</italic></td>
</tr>
<tr>
<td align="center" colspan="2"><bold><italic>Theme 2</italic>: <italic>PhD Supervision should be recognized in alternative assessments</italic></bold></td>
</tr>
<tr>
<td align="left" colspan="2">Senior researcher UMC 3:<break/>‘<italic>well</italic>, <italic>you know how things ought to be and show that too</italic>. <italic>And you straighten people out when they think they can deviate from it’</italic></td>
</tr>
<tr>
<td align="left" colspan="2">Senior researcher UMC 3: <break/><italic>‘not only have to invest in scientific training but in someone’s personal development in the wider sense of the term’</italic></td>
</tr>
<tr>
<td align="left" colspan="2">Senior researcher UMC 3:<break/><italic>‘you grow into this at a certain moment’ through the ‘interaction with your people’</italic></td>
</tr>
<tr>
<td align="left" colspan="2">Senior researcher UMC 3:<break/><italic>‘when you are trained well and when it is</italic>, <italic>kind of</italic>, <italic>explained to you’</italic></td>
</tr>
<tr>
<td align="center" colspan="2"><bold><italic>Theme 3</italic>: <italic>Teaching should be recognized in alternative assessments</italic></bold></td>
</tr>
<tr>
<td align="left" colspan="2">Junior researcher UMC 1: <break/><italic>‘To teach or to disseminate things to society</italic>: <italic>that does belong to it but is currently not considered to be a way to be a good scientist’</italic></td>
</tr>
<tr>
<td align="left" colspan="2">Senior researcher UMC 3: <break/>‘For some people, teaching is not their greatest hobby. And then they still do it and try to somehow get a good evaluation out of it’</td>
</tr>
<tr>
<td align="left" colspan="2">Senior researcher UMC 3: <break/><italic>‘You have to conduct research</italic>, <italic>and create societal relevant work and you have to be good at teaching too’</italic></td>
</tr>
<tr>
<td align="center" colspan="2"><bold>From Relational responsibilities to relational assessment</bold><break/><bold>Solutions</bold></td>
</tr>
<tr>
<td align="center" colspan="2">Research culture, formation of teams and activities of team leaders</td>
</tr>
<tr>
<td align="left"><italic>Quote</italic>:</td>
<td align="left"><italic>Topic</italic>:</td>
</tr>
<tr>
<td align="left">Senior researcher UMC 2:<break/><italic>‘In a research group</italic>, <italic>you function as a role model in that you also</italic>, <italic>well</italic>, <italic>know how things should go and show how things should go</italic>. <italic>And to call people to order in case they think they can deviate from this</italic>.<italic>’</italic></td>
<td align="left">The important role of team leaders on research culture</td>
</tr>
<tr>
<td align="left">Senior researcher UMC 2: <break/><italic>‘I would think that ‘responsible’ is more applicable to someone who intends to be of service to others and does not act in too selfish a way’</italic></td>
<td align="left">The important role of team leaders on research culture</td>
</tr>
<tr>
<td align="left"><italic>Junior Researchers UMC 1</italic>:<break/><italic>‘narcissism is an outgoing personality</italic>, <italic>that is not something compatible with responsible researchers</italic></td>
<td align="left">Relational responsibilities may not be compatible with certain personalities</td>
</tr>
<tr>
<td align="left">Senior researcher UMC 3:<break/><italic>‘Because it is almost the case that you remain a team player despite the system</italic>. <italic>Because the system holds individuals accountable of course’</italic></td>
<td align="left">The important role of formation of teams on research culture</td>
</tr>
<tr>
<td align="center" colspan="2"><bold><italic>Relational Assessment theme 1</italic>: <italic>Reward team effort and team science</italic></bold></td>
</tr>
<tr>
<td align="left" colspan="2"><italic>Quote Senior Researcher UMC 2</italic>:<break/><italic>‘According to me</italic>, <italic>the biggest bottleneck is that these criteria always apply to the individual</italic>. <italic>While we</italic>, <italic>in this conversation</italic>, <italic>emphasize that it is a collective effort</italic>, <italic>a team process’</italic></td>
</tr>
<tr>
<td align="center" colspan="2"><bold><italic>Relational assessment theme 2</italic>: <italic>provide 360 degrees of feedback</italic></bold></td>
</tr>
<tr>
<td align="left" colspan="2">Quote Senior research UMC 1:<break/><italic>‘perhaps people should be interviewed by commissions or something like that</italic>. <italic>That you get a way better picture of the person and how he operates in such a team’</italic></td>
</tr>
<tr>
<td align="center" colspan="2"><bold><italic>Relational assessment theme 3</italic>: <italic>engage in narrative assessment</italic></bold></td>
</tr>
<tr>
<td align="left" colspan="2">Quote Senior research UMC 1:<break/><italic>‘perhaps people should be interviewed by commissions or something like that</italic>. <italic>That you get a way better picture of the person and how he operates in such a team’</italic></td>
</tr>
<tr>
<td align="left" colspan="2">Senior Researcher UMC 2:<break/><italic>‘These soft skills are very difficult to capture in criteria that you can also put on a list’</italic>.</td>
</tr>
<tr>
<td align="center" colspan="2"><bold>Potential limitations of alternative assessments</bold></td>
</tr>
<tr>
<td align="left"><italic>Quote</italic> Senior researcher UMC 3:<break/><italic>‘shouldn’t you be afraid</italic>, <italic>at least I am</italic>, <italic>that we are going to design a whole system to get the non-responsible people in line</italic>? <italic>That the instrument is in fact a bit of an overkill</italic>?’</td>
<td align="left">Alternative assessment can also be a bureaucratic burden</td>
</tr>
<tr>
<td align="left"><italic>Quote</italic> Junior researcher UMC 1:<break/><italic>‘a little narcissism here and there is not so bad in itself because it will get you a long way and you can do many useful things with it in the end’</italic></td>
<td align="left">Individual recognition is not solely bad but may also have positive consequences</td>
</tr>
</tbody>
</table>
</alternatives>
</table-wrap>
<sec id="sec015">
<title>I. Responsible research as the relational responsibilities of researchers</title>
<p>Research integrity is a term that sparked lively discussions in the focus groups. But it is also a term that was difficult to capture as one ‘thing’; a thing that can be clearly defined and assessed. In the first part of the focus group, participants discussed what research integrity meant to them. They highlighted that a researcher with integrity is someone who does good research and knows what good research is. For many, research integrity is not necessarily a character trait that you develop–although honesty would be part of the requirements of a responsible researcher. More than a character trait, integrity is seen by participants as an element of good research practices associated with the replicability of someone’s work; the compliance with open science principles (such as sharing data); and collegial reviews to assess to what extent the published work is of high quality. Despite these elements, familiar ones for those professionally preoccupied with the topic, participants found it hard to make research integrity converge into a clear measure that could find its way in the assessment criteria. As a senior researcher (UMC 1) expressed it: <italic>And integrity</italic>: <italic>I cannot see yet how one should assess this</italic>. <italic>It is very important</italic>, <italic>of course</italic>, <italic>for a responsible scientist</italic>, <italic>to have some of these measures that you should ask for in a more qualitative manner or that should be part of the way people assess one another</italic>.</p>
<p>The reproducibility of scientific results, for instance, seems particularly hard to evaluate: ‘<italic>The only thing that comes to my mind that would be an unbiased</italic>, <italic>but really tricky</italic>, <italic>way to sort of assess honesty</italic>, <italic>is to look at how someone’s work is followed up and reproduced independently</italic>, <italic>right</italic>, <italic>after publication</italic>’ (Senior Researcher UMC 2).</p>
<p>But although research integrity seems elusive as a characteristic of research or of a researcher which one could directly measure and assess, participants did find indirect ways that open it up for evaluative purposes. What particularly stands out are the social relations and skills that are deemed necessary to act responsibly in research settings. These relations and skills are pivotal in conducting responsible research and permeate the kinds of role modeling and leadership that people are capable of. Or, in the words of a junior researcher, ‘<italic>being responsible for your researchers is also part of responsible research’</italic> (Junior researcher UMC 3). Junior researchers would appreciate more focus on scientists who act responsibly because <italic>‘people who enjoy the game</italic>, <italic>but that are not doing responsible research</italic>, <italic>they stick around too much</italic>’ (Junior researcher UMC 2). Another junior researcher sides with this point of view, though expressed somewhat stronger terms: <italic>‘When you look at the system as it is right now</italic>, <italic>it presorts quite significantly on a very specific</italic>, <italic>scary personality type’</italic> (Junior researcher UMC 1). A possible explanation for that presorting process, is that people who enjoy ‘the game’ sometimes become full professor predominantly on the basis of their skills as researcher, not so much on their skills as research leaders. Perhaps, these research content-related qualities should not stand out in the decision to let researchers ‘supervise these people who end up at home with a burn-out’ (Junior UMC 3). In addition to the inability to directly assess research integrity, the importance of social relations and skills in responsible research practices hence stood out as a second key finding. These relational responsibilities branch out in three different themes that should be (more) central in responsible research assessment according to our participants: researchers relate to peers in constructive collaboration to foster responsible research among researchers in biomedical research overall; researchers relate to PhD’s in responsible supervisory practices; and they relate to students in responsible teaching practices.</p>
<sec id="sec016">
<title>Relational responsibility theme 1: Collaboration</title>
<p>The participants agreed that collaboration in general is essential for scientific research. In biomedicine, research is mostly done in teams and the importance and value ‘team science’ has gained more attention recently [<xref ref-type="bibr" rid="pone.0307814.ref019">19</xref>, <xref ref-type="bibr" rid="pone.0307814.ref020">20</xref>]. Participants consider the ability to collaborate a pivotal skill for a biomedical researcher. They feel that poor collaborative skills are important reasons for disputes and potentially threaten research integrity. Having collaborative skills can, according to most of the participants, hence be considered as a core characteristic of a responsible researchers and therefore <italic>‘collaboration</italic>, <italic>the competence to collaborate</italic>, <italic>has to be assessed’</italic> (senior researcher UMC 3). The actual assessments of (the development of) this competence could increase the emphasis on the collaborative nature of biomedical research.</p>
<p>The question how to integrate this competence in individual assessments leads to various answers. As with supervision, one can ask for 360 degrees of feedback from team members to assess someone’s collaborative skills. Interestingly, some participants are geared towards more competitive inducements for collaboration: <italic>‘You could also have like</italic>, <italic>well</italic>, <italic>a prize for a well-functioning team; you could have something like a team player prize’</italic> (senior researcher UMC 3). Other participants, on the contrary, reckoned that rewarding collaboration skills through such external incentives might be problematic due to the fact that collaboration should remain an intrinsic drive of team leaders: ‘<italic>Well</italic>, <italic>many people have the inner incentive that they want their group to function well and we don’t need to have such an artificial score for it’</italic> (Senior Researcher UMC 1).</p>
</sec>
<sec id="sec017">
<title>Relational responsibility theme 2: PhD supervision</title>
<p>In addition to collaboration, the participants stress that PhD supervision is one of the key elements of responsible research practices. Although it is not valued and represented in their institutions’ assessment criteria, most participants feel that supervision is one of the cornerstones of such practices. To start with, they argue one has to be aware of the fact that one is a role model in their research group in the sense that ‘<italic>well</italic>, <italic>you know how things ought to be and show that too</italic>. <italic>And you straighten people out when they think they can deviate from it’</italic> (senior researcher UMC 3). In addition, participants consider that supervision of high quality should not solely focus on transmitting the technical knowledge and know-how that are essential for good research practices but should also focus on the relationship between supervisor and supervisee. They suggest that there should be attention to the personal development of the supervisee. As one senior researcher expressed it, to educate researchers to stand their ground in society you <italic>‘not only have to invest in scientific training but in someone’s personal development in the wider sense of the term’</italic> (senior researcher UMC 3). Such supervision skills require both training and practice to make people aware of their role as well as their capacities. Being promoted from junior to senior positions <italic>‘you grow into this at a certain moment’ through the ‘interaction with your people’</italic> (senior researcher UMC 3). And even though personality may play a role here as well, becoming a good role model and developing supervisory skills is easier <italic>‘when you are trained well and when it is</italic>, <italic>kind of</italic>, <italic>explained to you’</italic> (senior researcher UMC 3).</p>
<p>Assessment of supervision is complex: it is difficult to measure the quality of supervision and there are limitations to assess supervision quality by PhD students because they have a dependent relationship with their supervisor. A potential solution to these issues is also suggested. There can be a mentor system, where PI’s from other departments have regular meetings with PhD students, or an semi-anonymous ‘360 degrees assessment’ with special attention to supervision skills.</p>
</sec>
<sec id="sec018">
<title>Relational responsibility theme 3: Teaching</title>
<p>Currently, educational responsibilities are often seen as a burden for individual researchers because they are not a top priority for them. Although the education of students is a vital responsibility of universities, this is not automatically reflected into the ideals of good science and responsible research practices: <italic>‘To teach or to disseminate things to society</italic>: <italic>that does belong to it but is currently not considered to be a way to be a good scientist’</italic> (Junior researcher UMC 1). According to the participants, teaching has thus a very important indirect relation with responsible research practices. Students learn to become responsible researchers by following workshops and lectures, being supervised during master theses, develop academic skills in classrooms and gain experience in conducting research that is taught according to the highest standards of research integrity. However, engaging in education will cost researchers time and energy that they cannot spend on other ‘assessable’ tasks such as writing publications and grant applications. The notion that teaching will not be very important to their career advancement is widespread. This in turn makes that some participants have developed a pragmatic attitude towards teaching that can border on the opportunistic: <italic>‘For some people</italic>, <italic>teaching is not their greatest hobby</italic>. <italic>And then they still do it and try to somehow get a good evaluation out of it’</italic> (senior researcher UMC 3). What does not help, participants add, is that hiring committees seem to be looking for the impossible. As one of our senior researchers states: <italic>‘You have to conduct research</italic>, <italic>and create societal relevant work and you have to be good at teaching too’</italic> (Senior researcher UMC 3). But while you are expected to excel in all academic tasks, it is not easy to be good at everything.</p>
<p>The participants give several solutions to include education in the assessment system. This can be done by taking the hours spent on education, the development of new courses and student evaluations explicitly into consideration when assessing researchers. A possible limitation to these suggestions is also brought up: it may strengthen the pragmatic or opportunistic attitude to get high assessment scores on educational skills.</p>
</sec>
</sec>
<sec id="sec019">
<title>From relational responsibilities to relational assessment</title>
<p>The three relational responsibilities that our participants singled out are deemed essential in fostering responsible research practices and a positive research culture of a faculty, department, or research group. However, the participants acknowledge that relational responsibilities very much depend on research culture, formation of teams and activities of team leaders. They are key in engaging in, and maintaining, collaborative relations, supervise young researchers and are often considered a role model to give the right example in their team. <italic>‘In a research group</italic>, <italic>you function as a role model in that you also</italic>, <italic>well</italic>, <italic>know how things should go and show how things should go</italic>. <italic>And to call people to order in case they think they can deviate from this</italic>.<italic>’</italic> (Senior researcher UMC 2). The orientation towards the team and how it functions is bound up with an orientation towards the kinds of social skills, such as a reflexive stance and communicative skills that someone has to possess or develop. Some participants highlighted that what makes that someone becomes a team player is a specific attitude. As way of relating to others, taking relational responsibility is opposed to the search for self-aggrandizement that is also plentiful in academia: <italic>‘I would think that ‘responsible’ is more applicable to someone who intends to be of service to others and does not act in too selfish a way’</italic> (Senior researcher UMC 2). Or even more strongly put, responsible researchers can be opposed to narcistic attitudes of researchers: <italic>‘narcissism is an outgoing personality</italic>, <italic>that is not something compatible with responsible researchers</italic>.’ Though deemed essential to responsible research practices, team-oriented attitudes are not seen as core characteristics that are essential for researchers. If they want to further in their career, team oriented is something that is often being valued but never recognized in current evaluation forms: <italic>‘There are quite a lot of researchers who really make it</italic>, <italic>but who are not team players’</italic> (Junior Researcher UMC 2<italic>)</italic>. On the contrary, the ‘system’ seems to discourage team science by focusing on individual recognition: <italic>‘Because it is almost the case that you remain a team player despite the system</italic>. <italic>Because the system holds individuals accountable of course’</italic> (senior researcher UMC 3). Currently, the assessment criteria are predominantly focusing on achievements made by an individual. But when the research community deems relational responsibilities as important as individual achievement, then the assessment criteria should take such responsibilities into account. Below we describe the 3 themes that were highlighted by our participants as potential instruments to assess relational responsibilities.</p>
<sec id="sec020">
<title>Relational assessment theme 1: Reward team effort and team science</title>
<p>The current assessment criteria are mainly focused on individual publications and metrics performance and not rewarding team efforts. Since collaboration is considered one of the cornerstones of responsible biomedical research, more emphasis on team science in assessment and promotion criteria would help. This will also put less pressure on individual performance and reward a wider range of researchers for their contribution. Currently, the researchers who value collective performance as at least as important as individual achievement and may not focus solely at obtaining tenure or full professorship, do not feel recognized while they are very much involved in collaborative projects. As one researcher suggests: <italic>‘According to me</italic>, <italic>the biggest bottleneck is that these criteria always apply to the individual</italic>. <italic>While we</italic>, <italic>in this conversation</italic>, <italic>emphasize that it is a collective effort</italic>, <italic>a team process’</italic> (senior research UMC 3). A second way of better assessing teamwork would be to reward academic leaders who show that you can lead a flourishing team on several levels. Only if you can fulfill these criteria, you will have a chance to get promoted: ‘<italic>That you say something like</italic>: <italic>you only become a leader when you can really keep a team on track</italic>. <italic>And that is what you have to show</italic>. <italic>That is one of the competences that is assessed</italic>. <italic>And this means that you can manage this team</italic>, <italic>it means that you can see the broader scope of a research field</italic>…’ (senior researcher UMC 1)</p>
<p>A third and final suggestion would be to better reward a diverse set of roles in teams. This is not often part of current assessment systems but is a crucial factor in good team work. You need a diverse team with a wide variety of research qualities–and persons who possess such qualities–to make your research endeavor both responsible and successful. Rewarding team efforts here means that you have eyes for all the different perspectives and types work that are needed in a team and think of a way to make visible how an individual human being functions inside it: <italic>‘And to really give weight to this without that Hirsch-index type of thing</italic>. <italic>I don’t know how to exactly do that</italic>. <italic>So</italic>: <italic>recognize the diversity inside the team and your role in this’</italic> (senior researcher UMC 3).</p>
</sec>
<sec id="sec021">
<title>Relational assessment theme 2: Provide 360 degrees of feedback</title>
<p>In the focus group discussions, participants also addressed the advantages and disadvantages of the ‘360 degrees feedback’-model. On the positive side, this is a form of assessment where you can explore multiple dimensions of a person such as collaborative skills, motivation and personality by (anonymously) interviewing co-workers, subordinates, or bosses. One participant highlighted one of the benefits: <italic>‘my supervisor did not see all my qualities</italic>. <italic>Other ranks should have a better picture of your qualities and what you can improve</italic> (senior researcher UMC 3)’. A senior researcher, who is part of a committee that keeps track of PhD student well-being, comments upon the <italic>‘enormous number of dysfunctional labs</italic>, <italic>that you would never think of’</italic> (Senior researcher UMC 2). To get a grip on existing lab culture and the problems that might otherwise remain hidden, the participant advocates ‘the idea of interviewing also people who are junior’ to get unbiased and open feedback: ‘<italic>I mean</italic>, <italic>personally I also did this and it’s really interesting to see what people think of me</italic>, <italic>what I need to tone down</italic>, <italic>what I need to improve on</italic>, <italic>and I think that’s something that would be important in the assessment of senior PIs’</italic> (senior researcher UMC 2). Introducing 360 degrees of feedback would allow the institute to find out what people think of the leadership, the environment and the (lack of) transparency.</p>
<p>On the downside, it can potentially be influenced by negative attitudes of others who are begrudged, are unkind and can be subjective. Another limitation is the fact that it is time consuming for both the institution and the researcher to complete. Among the participants, this sparked the discussion on audits, a thorough (and time-consuming) way to explore responsible research practices on the shop floor by auditing research teams and practices in laboratories. The participants directly stated that for some disciplinary fields this would be easier to implement than other fields (e.g. audits may potentially have most impact in research labs, and may be less feasible for departments that conduct clinical research).</p>
</sec>
<sec id="sec022">
<title>Relational assessment theme 3: Engage in narrative assessment</title>
<p>A third way of assessing whether a researcher is responsible according to our participants is by using the narrative in assessment procedure. In narratives you assess the quality of a researcher in terms of research aims, individual motivation and responsible research practices and can answer the following questions. How does someone behave vis-à-vis others? What do colleagues say about the research or researcher? How compliant is their engagement with open science practices and what are the virtues of someone as a person? In line with what we remarked earlier, our participants were ambivalent about the idea that relational responsibilities can actually be directly assessed as they are not so easy to detect or measure. Or, as one senior researcher expressed it: <italic>‘These soft skills are very difficult to capture in criteria that you can also put on a list’</italic> (Senior Researcher UMC 2). But assessment in a narrative form could help out here: <italic>‘perhaps people should be interviewed by commissions or something like that</italic>. <italic>That you get a way better picture of the person and how he operates in such a team’</italic> (Senior research UMC 1). Another participant pointed to the possibilities of a narrative assessment in differentiating between qualities someone possesses. This comes back to the notion that you will not have a one size, fits all assessment and that most researchers have different qualities. Some researchers in specific disciplines do not publish much but have other important academic qualities that should be acknowledged and considered. To get a hold on these qualities, a narrative can shine light on them. Only with this, you will get a better idea what the researcher is like and what his qualities are.</p>
<p>Although relational responsibilities are valued as part of responsible research behavior and practices, some ambivalence remains here. Some participants also raised concerns that these alternative assessments would lead to too much assessment procedures for researchers: <italic>‘shouldn’t you be afraid</italic>, <italic>at least I am</italic>, <italic>that we are going to design a whole system to get the non-responsible people in line</italic>? <italic>That the instrument is in fact a bit of an overkill</italic>?’ (senior researcher UMC 3). Other participants do not see the strife for recognition from others as an evil per se. They argue that <italic>‘a little narcissism here and there is not so bad in itself because it will get you a long way and you can do many useful things with it in the end’</italic> (junior researcher UMC 1). This point of view is in a certain way partially incongruent with other findings as this does emphasize the need for individual recognition and fewer relational responsibilities from narcissists.</p>
</sec>
</sec>
</sec>
<sec id="sec023" sec-type="conclusions">
<title>Discussion</title>
<p>In general, current evaluation criteria in academia have received severe criticism in the past years [<xref ref-type="bibr" rid="pone.0307814.ref004">4</xref>–<xref ref-type="bibr" rid="pone.0307814.ref007">7</xref>, <xref ref-type="bibr" rid="pone.0307814.ref009">9</xref>, <xref ref-type="bibr" rid="pone.0307814.ref013">13</xref>]. Several initiatives have pleaded for a different approach when assessing research and researchers and often conventional metrics, such as the impact factor or the H-index are considered weak and unidimensional. However, the use of these metrics is widespread and engrained in academic culture [<xref ref-type="bibr" rid="pone.0307814.ref005">5</xref>]. In this study we highlighted what is considered responsible research among junior and senior researchers in the Netherlands and how this can be assessed in formal assessment criteria. The participants of the focus groups reflected on responsible research and highlighted several academic responsibilities (such as supervision, collaboration and teaching) that are often overlooked and that are considered a crucial prerequisite for responsible research. After our systematic analysis of these ‘relational responsibilities’, we focused on the solutions that the participants came up with: how can these duties be reflected in multidimensional, concrete and sustainable assessment criteria? Participants emphasized the importance of assessing team science, suggested the use of a narrative in researcher assessment and valued the use of 360 degrees assessment of researchers. Participants believed that these alternative assessments, centered on relational responsibilities, could help in fostering responsible research practices.</p>
<p>Our study sheds a new light on the assessment discussions as it is emphasizing the assessment of relational responsibilities. Besides, our study also provides a number of suggestions how these skills should be assessed and how these skills are closely related and connected to responsible research. Our findings also fit well with the national initiative in the Netherlands entitled ‘<italic>room for everyone’s talent’</italic> that is initiated by the Dutch Royal Academy of Science and recognized by all universities, funding agencies and other stakeholders in the Netherlands. In their position paper, they plead for a radical change in assessing researchers with an emphasis on diversity, focus on quality, stimulating open science and encouraging academic leadership [<xref ref-type="bibr" rid="pone.0307814.ref021">21</xref>]. Although not specifically focusing on relational responsibilities as a different entity, the way these responsibilities should be recognized is reflected in their position paper.</p>
<sec id="sec024">
<title>Interpretation of data</title>
<p>Our study identifies the need for alternative assessment criteria that are related to responsible research practices and outlines how relational responsibilities should be valued in the assessment of research and researchers. Our study proposes alternatives that, although they may be a bit more difficult to implement and requires that assessment procedures are revised so as to enable them to incorporate qualitative evaluations. This is possibly challenging in its novelty, and requires additional research to further develop these evaluations. Moreover, the suggestions from the participants can help policy makers to redesign their current assessment criteria. They also are a plea for more elaborate discussions on the criteria and invite policymakers to use a different strategy to assess researchers. For example, 360 degree evaluations can form the basis of the yearly evaluation conversations where researchers have to reflect on the responsibilities they have towards others–not just their achievements. Before we turn to the introduction of these new forms of assessment in institutional settings it is necessary to emphasize the limitations of these assessment themes. There are (at least) three.</p>
<p>First, our results are still a first step and these types of assessment criteria are still in its infancy and need rigorous testing before they can be put into practice. Furthermore, the participants recommend a blended approach in which both the conventional criteria (JIF, H-index, funding acquisition) as the new criteria should inform assessment of researchers. We thus should exercise caution to avoid discarding valuable elements or aspects when making comprehensive changes or judgments, as prematurely eliminating the essential along with the dispensable may lead to unintended consequences.</p>
<p>Second, taking disciplinary differences into account, we are aware that we only included biomedical researchers. However, we think that the emphasized relational responsibilities such as leadership, collaboration or supervision are not skills that are solely displayed in biomedical research and most likely the results can be generalized to other disciplines.</p>
<p>Third, it is good to reflect on both reliability and validity of our findings in other research contexts (both disciplinary fields and other countries) since we only included 3 Dutch UMCs in our analysis. Future research could address these limitations by including other disciplines in our methods when designing and testing novel assessment criteria, systematically ask for reasons for non-response and try to organize larger focus groups in other disciplines.</p>
<p>Finally, the results also underline the relation between relational responsibilities and responsible research. Earlier research concluded that responsible research is strongly shaped by research cultures. In these cultures, norms and unwritten rules play a significant role in how researchers behave and which social skills are encouraged [<xref ref-type="bibr" rid="pone.0307814.ref021">21</xref>–<xref ref-type="bibr" rid="pone.0307814.ref024">24</xref>]. Assessing these skills may have an impact on more responsible research practices on the shop floor.</p>
<p>We also would like to highlight the CRediT taxonomy initiative as an initiative to improve researcher assessment. The CrediT (Contributor Roles Taxonomy) is a classification system developed to provide standardized categories for describing the contributions of individuals to scholarly work. It was created to address the need for more transparent and granular acknowledgment of individual contributions within scholarly publications. The taxonomy outlines various roles that individuals may play in the research process, going beyond traditional authorship credits. This is a better way to give credit where credit is due and further details on the roles authors play and thus give better insight in roles that can help in the assessment of researchers. Another welcome initiative is CoARA, the Coalition for advancing Research Assessment. This European coalition tries to change the current assessment system by improving the assessment of research, researchers and research organizations by recognizing and rewarding the diverse outputs, practices and activities that maximize the quality and impact of research. This requires basing assessment primarily on qualitative judgement, for which peer review is central, supported by responsible use of quantitative indicators.</p>
</sec>
<sec id="sec025">
<title>Evaluation labs</title>
<p>Since our project only explored the perspective of biomedical researchers, we wanted to corroborate whether these newly formulated evaluation ideas actually can be implemented in real life settings. Therefore, we conducted two evaluation labs. In these labs, we invited stakeholders (policy makers working at funding bodies in the Netherlands, and policy makers working at the eight University medical centers) to discuss the results of our study and let them reflect on whether the suggestions are implementable and feasible. They were encouraged to think what solutions that were highlighted by the participants of the focus groups could actually be useful for their institution. We asked them to discuss whether and how the themes from the focus group study could help shape new assessment criteria in their institution (exercise 1). After the discussion of the themes, we asked them to rank the themes on importance and feasibility (exercise 2).</p>
<sec id="sec026">
<title>Exercise 1</title>
<p>The participants think that integrity should be an overall assessment criterium that should be included in all themes and included the work environment as part of research integrity. They agree that assessing <italic>relational responsibilities</italic> remains a very complicated matter as the participants consider assessing them as something subjective and incomplete. However, they do consider it potentially interesting to assess leadership skills as relational responsibility. They specifically highlighted supervision, and the participants suggested that PhD students should evaluate the supervisor and this knowledge should be available to include in promotion and assessment.</p>
<p>As for assessing relational responsibilities, they suggest that <italic>360 degree assessment</italic> can be an interesting tool and should be expanded to international colleagues as well. Furthermore, they suggest to include the evaluation of the researchers by their patients and students as well. Finally, the assessment by <italic>narrative</italic> has the potential to be important. The main concern is that it may expand the workload of researchers even more with administrative work. It is also prone for opportunistic behaviour and manipulation. Besides, achieving trust in the narrative-method can be challenging.</p>
</sec>
<sec id="sec027">
<title>Exercise 2</title>
<p>We asked the participants to rank the themes by whether they consider this the most essential theme. After careful deliberation, the participants ranked collaboration, integrity and supervision as most essential. After this exercise, they included diversity as an important theme that is extensively discussed in the past years and should be considered as an evaluation criterion. Research groups can benefit from diversity with respect to competences and qualities.</p>
<p>Two main concerns remain in place according to the participants. In their discussions they repeatedly highlighted that there is a natural overlap between the teams and some themes are also covered by other themes (e.g. Is good supervision part of research integrity?). They also noted that scientific rigor and quality are not assessed anywhere and suggested that this should be operationalized as they relate closely with responsible research and are connected with the research integrity theme.</p>
</sec>
</sec>
<sec id="sec028">
<title>Strengths and limitations</title>
<p>There are some strengths and limitations to this study to bear in mind. First of all, this is the first study that investigates responsible research as a criterion in assessment of researchers in a qualitative way. Qualitative methods are helpful if you want to explore new perspectives or formulate new hypothesis and offers a complimentary contribution to prior quantitative and hypothesis driven research. Secondly, the participants highlight that it is possible to assess relational responsibilities. Since the current practices predominantly rely on quantitative output criteria, this is an important message that may help policymakers to adopt alternative criteria in their assessment procedures. Finally, we were able to test the results of the focus groups in the evaluation labs. This has resulted in interesting thoughts on potential implementation issues, but shine also new light on the discussion of what is actually possible, warranted and needed for policy makers and funders.</p>
<p>Our study also comes with some limitations. First, we only included biomedical researchers. Although we are convinced that most criteria can be generalized to other disciplinary fields, one may argue that in other disciplines (e.g. the humanities) slightly different assessment criteria might apply. Biomedicine was purposefully considered as a strategic research site. Evidence suggests that biomedical research assessment is prone for quantification [<xref ref-type="bibr" rid="pone.0307814.ref025">25</xref>]. Also, biomedicine is the largest discipline in the Netherlands (and abroad). Besides, dominant organisational forms in biomedical research—research projects, groups and labs—are increasingly found in the social sciences and humanities as well [<xref ref-type="bibr" rid="pone.0307814.ref026">26</xref>, <xref ref-type="bibr" rid="pone.0307814.ref027">27</xref>]. Our findings will thus have implications beyond biomedicine. Future research is thus needed to explore these potential disciplinary differences and should test the proposed assessment criteria. Second, a form of response bias cannot be ruled out. We have sent out invitations to a large set of participants. Most of the invited researchers were asked for the reason for non-participation. Most often, lack of time was the main reason not to participate. Third, it could be that we are still missing perspectives. Some focus groups were rather small and although we reached data saturation for most themes, it could well be that some perspectives are not highlighted.</p>
</sec>
<sec id="sec029">
<title>Changing the system</title>
<p>It is a daunting task to change assessment criteria that have been used for decades in various institutions. The academic enterprise is a complex system and change is often met with a certain skepticism, both by established researchers that have made careers by older assessment criteria (survivor bias) and by the fear that future assessment criteria will change academia into a system where only ‘soft skills’ are assessed. However, change starts with awareness and it helps that there are successful initiatives that have shown their value. Besides, implementation has started by some of the most important stakeholders. From our results we draw the conclusion that responsible research, constituted of both relational responsibilities and more quantifiable assessment criteria, should be a major element of researchers assessment in the future. This entails both 1) rigorous testing of reformed assessment criteria before implementing them in practice and 2) communicating this transparently with researchers, so they know what to expect in future assessments. More attention is still needed on relational responsibilities that, taken into account that they are highly influencing responsible research, should therefore play a more prominent role in future assessment criteria.</p>
<p>In conclusion, our study suggests that relational responsibilities should ideally play a more prominent role in future assessment criteria as these relational responsibilities are reckoned beneficial for the practice of responsible research. Our participants gave several suggestions on how to make these skills measurable and assessable in future assessment criteria. However, the development of these criteria is still in its infancy and future research should focus on how to make these criteria more tangible, concrete and applicable in daily practice.</p>
</sec>
</sec>
<sec id="sec030" sec-type="supplementary-material">
<title>Supporting information</title>
<supplementary-material id="pone.0307814.s001" mimetype="application/pdf" position="float" xlink:href="info:doi/10.1371/journal.pone.0307814.s001" xlink:type="simple">
<label>S1 File</label>
<caption>
<title>Medical ethical review Approval_METC_Tijdink.</title>
<p>(PDF)</p>
</caption>
</supplementary-material>
<supplementary-material id="pone.0307814.s002" mimetype="application/vnd.openxmlformats-officedocument.wordprocessingml.document" position="float" xlink:href="info:doi/10.1371/journal.pone.0307814.s002" xlink:type="simple">
<label>S1 Table</label>
<caption>
<title>Translation Dutch quotes focus groups.</title>
<p>(DOCX)</p>
</caption>
</supplementary-material>
</sec>
</body>
<back>
<ref-list>
<title>References</title>
<ref id="pone.0307814.ref001"><label>1</label><mixed-citation publication-type="journal" xlink:type="simple"><name name-style="western"><surname>De Rond</surname> <given-names>M</given-names></name>, <name name-style="western"><surname>Miller</surname> <given-names>AN</given-names></name>. <article-title>Publish or Perish: Bane or Boon of Academic Life?</article-title> <source>Journal of Management Inquiry</source>. <year>2005</year>;<volume>14</volume>(<issue>4</issue>):<fpage>321</fpage>–<lpage>9</lpage>.</mixed-citation></ref>
<ref id="pone.0307814.ref002"><label>2</label><mixed-citation publication-type="journal" xlink:type="simple"><name name-style="western"><surname>Tijdink</surname> <given-names>JK</given-names></name>. <article-title>[Publish &amp; Perish; research on research and researchers].</article-title> <source>Tijdschr Psychiatr.</source> <year>2017</year><issue>;</issue><volume>59</volume>(<issue>7</issue>):<fpage>406</fpage>–<lpage>13</lpage>.</mixed-citation></ref>
<ref id="pone.0307814.ref003"><label>3</label><mixed-citation publication-type="journal" xlink:type="simple"><name name-style="western"><surname>Moher</surname> <given-names>D</given-names></name>, <name name-style="western"><surname>Naudet</surname> <given-names>F</given-names></name>, <name name-style="western"><surname>Cristea</surname> <given-names>IA</given-names></name>, <name name-style="western"><surname>Miedema</surname> <given-names>F</given-names></name>, <name name-style="western"><surname>Ioannidis</surname> <given-names>JPA</given-names></name>, <name name-style="western"><surname>Goodman</surname> <given-names>SN</given-names></name>. <article-title>Assessing scientists for hiring, promotion, and tenure</article-title>. <source>PLOS Biology</source>. <year>2018</year>;<volume>16</volume>(<issue>3</issue>):<fpage>e2004089</fpage>. <comment>doi: <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.1371/journal.pbio.2004089" xlink:type="simple">10.1371/journal.pbio.2004089</ext-link></comment> <object-id pub-id-type="pmid">29596415</object-id></mixed-citation></ref>
<ref id="pone.0307814.ref004"><label>4</label><mixed-citation publication-type="journal" xlink:type="simple"><name name-style="western"><surname>Moher</surname> <given-names>D</given-names></name>, <name name-style="western"><surname>Goodman</surname> <given-names>SN</given-names></name>, <name name-style="western"><surname>Ioannidis</surname> <given-names>JP</given-names></name>. <article-title>Academic criteria for appointment, promotion and rewards in medical research: where’s the evidence?</article-title> <source>Eur J Clin Invest</source>. <year>2016</year>;<volume>46</volume>(<issue>5</issue>):<fpage>383</fpage>–<lpage>5</lpage>. <comment>doi: <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.1111/eci.12612" xlink:type="simple">10.1111/eci.12612</ext-link></comment> <object-id pub-id-type="pmid">26924551</object-id></mixed-citation></ref>
<ref id="pone.0307814.ref005"><label>5</label><mixed-citation publication-type="journal" xlink:type="simple"><name name-style="western"><surname>Wilsdon</surname> <given-names>J.</given-names></name> <article-title>The Metric Tide: Independent Review of the Role of Metrics in Research Assessment and Management</article-title>. <source>55 City Road, London</source><year>2015</year>. Available from: <ext-link ext-link-type="uri" xlink:href="https://sk.sagepub.com/books/the-metric-tide" xlink:type="simple">https://sk.sagepub.com/books/the-metric-tide</ext-link>.</mixed-citation></ref>
<ref id="pone.0307814.ref006"><label>6</label><mixed-citation publication-type="journal" xlink:type="simple"><name name-style="western"><surname>Hicks</surname> <given-names>D</given-names></name>, <name name-style="western"><surname>Wouters</surname> <given-names>P</given-names></name>, <name name-style="western"><surname>Waltman</surname> <given-names>L</given-names></name>, <name name-style="western"><surname>de Rijcke</surname> <given-names>S</given-names></name>, <name name-style="western"><surname>Rafols</surname> <given-names>I</given-names></name>. <article-title>Bibliometrics: The Leiden Manifesto for research metrics</article-title>. <source>Nature</source>. <year>2015</year>;<volume>520</volume>(<issue>7548</issue>):<fpage>429</fpage>–<lpage>31</lpage>. <comment>doi: <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.1038/520429a" xlink:type="simple">10.1038/520429a</ext-link></comment> <object-id pub-id-type="pmid">25903611</object-id></mixed-citation></ref>
<ref id="pone.0307814.ref007"><label>7</label><mixed-citation publication-type="journal" xlink:type="simple"><name name-style="western"><surname>Rice</surname> <given-names>DB</given-names></name>, <name name-style="western"><surname>Raffoul</surname> <given-names>H</given-names></name>, <name name-style="western"><surname>Ioannidis</surname> <given-names>JPA</given-names></name>, <name name-style="western"><surname>Moher</surname> <given-names>D</given-names></name>. <article-title>Academic criteria for promotion and tenure in biomedical sciences faculties: cross sectional analysis of international sample of universities</article-title>. <source>BMJ</source>. <year>2020</year>;<volume>369</volume>:<fpage>m2081</fpage>. <comment>doi: <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.1136/bmj.m2081" xlink:type="simple">10.1136/bmj.m2081</ext-link></comment> <object-id pub-id-type="pmid">32586791</object-id></mixed-citation></ref>
<ref id="pone.0307814.ref008"><label>8</label><mixed-citation publication-type="journal" xlink:type="simple"><name name-style="western"><surname>Schimanski</surname> <given-names>LA</given-names></name>, <name name-style="western"><surname>Alperin</surname> <given-names>JP</given-names></name>. <article-title>The evaluation of scholarship in academic promotion and tenure processes: Past, present, and future.</article-title> <source>F1000Res</source>. <year>2018</year> <month>Oct</month> <day>5</day>;<volume>7</volume>:<fpage>1605</fpage>. <comment>doi: <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.12688/f1000research.16493.1" xlink:type="simple">10.12688/f1000research.16493.1</ext-link></comment> <object-id pub-id-type="pmid">30647909</object-id></mixed-citation></ref>
<ref id="pone.0307814.ref009"><label>9</label><mixed-citation publication-type="journal" xlink:type="simple"><name name-style="western"><surname>Schmid</surname> <given-names>SL</given-names></name>. <article-title>Five years post-DORA: promoting best practices for research assessment</article-title>. <source>Mol Biol Cell</source>. <year>2017</year>;<volume>28</volume>(<issue>22</issue>):<fpage>2941</fpage>–<lpage>4</lpage>. <comment>doi: <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.1091/mbc.E17-08-0534" xlink:type="simple">10.1091/mbc.E17-08-0534</ext-link></comment> <object-id pub-id-type="pmid">29084913</object-id></mixed-citation></ref>
<ref id="pone.0307814.ref010"><label>10</label><mixed-citation publication-type="journal" xlink:type="simple"><name name-style="western"><surname>Meirmans</surname> <given-names>S</given-names></name>, <name name-style="western"><surname>Butlin</surname> <given-names>RK</given-names></name>, <name name-style="western"><surname>Charmantier</surname> <given-names>A</given-names></name>, <name name-style="western"><surname>Engelstädter</surname> <given-names>J</given-names></name>, <name name-style="western"><surname>Groot</surname> <given-names>AT</given-names></name>, <name name-style="western"><surname>King</surname> <given-names>KC</given-names></name>, <etal>et al</etal>. <article-title>Science policies: How should science funding be allocated? An evolutionary biologists’ perspective</article-title>. <source>Journal of Evolutionary Biology</source>. <year>2019</year>;<volume>32</volume>(<issue>8</issue>):<fpage>754</fpage>–<lpage>68</lpage>. <comment>doi: <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.1111/jeb.13497" xlink:type="simple">10.1111/jeb.13497</ext-link></comment> <object-id pub-id-type="pmid">31215105</object-id></mixed-citation></ref>
<ref id="pone.0307814.ref011"><label>11</label><mixed-citation publication-type="journal" xlink:type="simple"><name name-style="western"><surname>Horbach</surname> <given-names>S</given-names></name>, <name name-style="western"><surname>Bouter</surname> <given-names>LM</given-names></name>, <name name-style="western"><surname>Gaskell</surname> <given-names>G</given-names></name>, <name name-style="western"><surname>Hiney</surname> <given-names>M</given-names></name>, <name name-style="western"><surname>Kavouras</surname> <given-names>P</given-names></name>, <name name-style="western"><surname>Mejlgaard</surname> <given-names>N</given-names></name>, <etal>et al</etal>. <article-title>Designing and implementing a research integrity promotion plan: Recommendations for research funders</article-title>. <source>PLoS Biol</source>. <year>2022</year>;<volume>20</volume>(<issue>8</issue>):<fpage>e3001773</fpage>. <comment>doi: <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.1371/journal.pbio.3001773" xlink:type="simple">10.1371/journal.pbio.3001773</ext-link></comment> <object-id pub-id-type="pmid">35984842</object-id></mixed-citation></ref>
<ref id="pone.0307814.ref012"><label>12</label><mixed-citation publication-type="journal" xlink:type="simple"><name name-style="western"><surname>Moher</surname> <given-names>D</given-names></name>, <name name-style="western"><surname>Bouter</surname> <given-names>L</given-names></name>, <name name-style="western"><surname>Kleinert</surname> <given-names>S</given-names></name>, <name name-style="western"><surname>Glasziou</surname> <given-names>P</given-names></name>, <name name-style="western"><surname>Sham</surname> <given-names>MH</given-names></name>, <name name-style="western"><surname>Barbour</surname> <given-names>V</given-names></name>, <etal>et al</etal>. <article-title>The Hong Kong Principles for assessing researchers: Fostering research integrity</article-title>. <source>PLoS Biol</source>. <year>2020</year>;<volume>18</volume>(<issue>7</issue>):<fpage>e3000737</fpage>. <comment>doi: <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.1371/journal.pbio.3000737" xlink:type="simple">10.1371/journal.pbio.3000737</ext-link></comment> <object-id pub-id-type="pmid">32673304</object-id></mixed-citation></ref>
<ref id="pone.0307814.ref013"><label>13</label><mixed-citation publication-type="journal" xlink:type="simple"><name name-style="western"><surname>Ioannidis</surname> <given-names>JP</given-names></name>, <name name-style="western"><surname>Greenland</surname> <given-names>S</given-names></name>, <name name-style="western"><surname>Hlatky</surname> <given-names>MA</given-names></name>, <name name-style="western"><surname>Khoury</surname> <given-names>MJ</given-names></name>, <name name-style="western"><surname>Macleod</surname> <given-names>MR</given-names></name>, <name name-style="western"><surname>Moher</surname> <given-names>D</given-names></name>, <etal>et al</etal>. <article-title>Increasing value and reducing waste in research design, conduct, and analysis</article-title>. <source>Lancet</source>. <year>2014</year>;<volume>383</volume>(<issue>9912</issue>):<fpage>166</fpage>–<lpage>75</lpage>. <comment>doi: <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.1016/S0140-6736%2813%2962227-8" xlink:type="simple">10.1016/S0140-6736(13)62227-8</ext-link></comment> <object-id pub-id-type="pmid">24411645</object-id></mixed-citation></ref>
<ref id="pone.0307814.ref014"><label>14</label><mixed-citation publication-type="journal" xlink:type="simple"><name name-style="western"><surname>Wolff</surname> <given-names>C</given-names></name>, <name name-style="western"><surname>Rod</surname> <given-names>AB</given-names></name>, <name name-style="western"><surname>Schonfeld</surname> <given-names>R</given-names></name>. <source>UK survey of academics 2015</source>. <year>2016</year>.</mixed-citation></ref>
<ref id="pone.0307814.ref015"><label>15</label><mixed-citation publication-type="journal" xlink:type="simple"><name name-style="western"><surname>Robinson-Garcia</surname> <given-names>N</given-names></name>, <name name-style="western"><surname>Costas</surname> <given-names>R</given-names></name>, <name name-style="western"><surname>Nane</surname> <given-names>GF</given-names></name>, <name name-style="western"><surname>van Leeuwen</surname> <given-names>TN</given-names></name>. <article-title>Valuation regimes in academia: Researchers’ attitudes towards their diversity of activities and academic performance.</article-title> <source>Research Evaluation</source>. <year>2023</year>;<volume>32</volume>(<issue>2</issue>):<fpage>496</fpage>–<lpage>514</lpage></mixed-citation></ref>
<ref id="pone.0307814.ref016"><label>16</label><mixed-citation publication-type="journal" xlink:type="simple"><name name-style="western"><surname>Fochler</surname> <given-names>M</given-names></name>, <name name-style="western"><surname>Felt</surname> <given-names>U</given-names></name>, <name name-style="western"><surname>Müller</surname> <given-names>R</given-names></name>. <article-title>Unsustainable Growth, Hyper-Competition, and Worth in Life Science Research: Narrowing Evaluative Repertoires in Doctoral and Postdoctoral Scientists’ Work and Lives.</article-title> <source>Minerva</source>. <year>2016</year>;<volume>54</volume>:<fpage>175</fpage>–<lpage>200</lpage>.14. <comment>doi: <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.1007/s11024-016-9292-y" xlink:type="simple">10.1007/s11024-016-9292-y</ext-link></comment> <object-id pub-id-type="pmid">27340295</object-id></mixed-citation></ref>
<ref id="pone.0307814.ref017"><label>17</label><mixed-citation publication-type="journal" xlink:type="simple"><name name-style="western"><surname>Morse</surname> <given-names>JM</given-names></name>, <name name-style="western"><surname>Barrett</surname> <given-names>M</given-names></name>, <name name-style="western"><surname>Mayan</surname> <given-names>M</given-names></name>, <name name-style="western"><surname>Olson</surname> <given-names>K</given-names></name>, <name name-style="western"><surname>Spiers</surname> <given-names>J</given-names></name>. <article-title>Verification Strategies for Establishing Reliability and Validity in Qualitative Research.</article-title> <source>International Journal of Qualitative Methods</source>. <year>2002</year>;<volume>1</volume>(<issue>2</issue>):<fpage>13</fpage>–<lpage>22</lpage>.</mixed-citation></ref>
<ref id="pone.0307814.ref018"><label>18</label><mixed-citation publication-type="journal" xlink:type="simple"><name name-style="western"><surname>Elo</surname> <given-names>S</given-names></name>, <name name-style="western"><surname>Kyngäs</surname> <given-names>H</given-names></name>. <article-title>The qualitative content analysis process.</article-title> <source>J Adv Nurs</source>. <year>2008</year>;<volume>62</volume>(<issue>1</issue>):<fpage>107</fpage>–<lpage>15</lpage>. <comment>doi: <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.1111/j.1365-2648.2007.04569.x" xlink:type="simple">10.1111/j.1365-2648.2007.04569.x</ext-link></comment> <object-id pub-id-type="pmid">18352969</object-id></mixed-citation></ref>
<ref id="pone.0307814.ref019"><label>19</label><mixed-citation publication-type="journal" xlink:type="simple"><name name-style="western"><surname>Lincoln</surname> <given-names>YS</given-names></name>, <name name-style="western"><surname>Guba</surname> <given-names>EG</given-names></name>, <name name-style="western"><surname>Pilotta</surname> <given-names>JJ</given-names></name>. <article-title>Naturalistic inquiry: Beverly Hills, CA: Sage Publications, 1985, 416 pp., $25.00 (Cloth).</article-title> <source>International Journal of Intercultural Relations</source>. <year>1985</year>;<volume>9</volume>(<issue>4</issue>):<fpage>438</fpage>–<lpage>9</lpage>.</mixed-citation></ref>
<ref id="pone.0307814.ref020"><label>20</label><mixed-citation publication-type="journal" xlink:type="simple"><name name-style="western"><surname>Hall</surname> <given-names>KL</given-names></name>, <name name-style="western"><surname>Vogel</surname> <given-names>AL</given-names></name>, <name name-style="western"><surname>Huang</surname> <given-names>GC</given-names></name>, <name name-style="western"><surname>Serrano</surname> <given-names>KJ</given-names></name>, <name name-style="western"><surname>Rice</surname> <given-names>EL</given-names></name>, <name name-style="western"><surname>Tsakraklides</surname> <given-names>SP</given-names></name>, <etal>et al</etal>. <article-title>The science of team science: A review of the empirical evidence and research gaps on collaboration in science</article-title>. <source>American Psychologist</source>. <year>2018</year>;<volume>73</volume>:<fpage>532</fpage>–<lpage>48</lpage>. <comment>doi: <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.1037/amp0000319" xlink:type="simple">10.1037/amp0000319</ext-link></comment> <object-id pub-id-type="pmid">29792466</object-id></mixed-citation></ref>
<ref id="pone.0307814.ref021"><label>21</label><mixed-citation publication-type="journal" xlink:type="simple"><collab>VSNU</collab>. <article-title>Room for everyone’s talent—Towards a new balance in the recognition and rewards of academics.</article-title> <source>Netherlands</source>; <year>2019</year>.</mixed-citation></ref>
<ref id="pone.0307814.ref022"><label>22</label><mixed-citation publication-type="journal" xlink:type="simple"><name name-style="western"><surname>Haven</surname> <given-names>T</given-names></name>, <name name-style="western"><surname>Bouter</surname> <given-names>L</given-names></name>, <name name-style="western"><surname>Mennen</surname> <given-names>L</given-names></name>, <name name-style="western"><surname>Tijdink</surname> <given-names>J</given-names></name>. <article-title>Superb supervision: A pilot study on training supervisors to convey responsible research practices onto their PhD candidates.</article-title> <source>Account Res.</source> <year>2022</year>:1- <comment>doi: <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.1080/08989621.2022.2071153" xlink:type="simple">10.1080/08989621.2022.2071153</ext-link></comment> <object-id pub-id-type="pmid">35475492</object-id></mixed-citation></ref>
<ref id="pone.0307814.ref023"><label>23</label><mixed-citation publication-type="journal" xlink:type="simple"><name name-style="western"><surname>Valkenburg</surname> <given-names>G</given-names></name>, <name name-style="western"><surname>Dix</surname> <given-names>G</given-names></name>, <name name-style="western"><surname>Tijdink</surname> <given-names>J</given-names></name>, <name name-style="western"><surname>de Rijcke</surname> <given-names>S</given-names></name>. <article-title>Expanding Research Integrity: A Cultural-Practice Perspective.</article-title> <source>Sci Eng Ethics</source>. <year>2021</year>;<volume>27</volume>(<issue>1</issue>):<fpage>10</fpage>. <comment>doi: <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.1007/s11948-021-00291-z" xlink:type="simple">10.1007/s11948-021-00291-z</ext-link></comment> <object-id pub-id-type="pmid">33559767</object-id></mixed-citation></ref>
<ref id="pone.0307814.ref024"><label>24</label><mixed-citation publication-type="journal" xlink:type="simple"><name name-style="western"><surname>Valkenburg</surname> <given-names>G</given-names></name>, <name name-style="western"><surname>Dix</surname> <given-names>G</given-names></name>, <name name-style="western"><surname>Tijdink</surname> <given-names>J</given-names></name>, <name name-style="western"><surname>de Rijcke</surname> <given-names>S</given-names></name>. <article-title>Making researchers responsible: attributions of responsibility and ambiguous notions of culture in research codes of conduct.</article-title> <source>BMC Med Ethics</source>. <year>2020</year>;<volume>21</volume>(<issue>1</issue>):<fpage>56</fpage>. <comment>doi: <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.1186/s12910-020-00496-0" xlink:type="simple">10.1186/s12910-020-00496-0</ext-link></comment> <object-id pub-id-type="pmid">32635905</object-id></mixed-citation></ref>
<ref id="pone.0307814.ref025"><label>25</label><mixed-citation publication-type="journal" xlink:type="simple"><name name-style="western"><surname>Rushforth</surname> <given-names>A</given-names></name>, <name name-style="western"><surname>de Rijcke</surname> <given-names>S</given-names></name>. <article-title>Accounting for Impact? The Journal Impact Factor and the Making of Biomedical Research in the Netherlands</article-title>. <source>Minerva</source>. <year>2015</year>;<volume>53</volume>(<issue>2</issue>):<fpage>117</fpage>–<lpage>39</lpage>. <comment>doi: <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.1007/s11024-015-9274-5" xlink:type="simple">10.1007/s11024-015-9274-5</ext-link></comment> <object-id pub-id-type="pmid">26097258</object-id></mixed-citation></ref>
<ref id="pone.0307814.ref026"><label>26</label><mixed-citation publication-type="journal" xlink:type="simple"><name name-style="western"><surname>Felt</surname> <given-names>U</given-names></name>. <article-title>Knowing and Living in Academic Research</article-title>. <source>Convergence and Heterogeneity in Research Cultures in the European Context</source> <year>2009</year>.</mixed-citation></ref>
<ref id="pone.0307814.ref027"><label>27</label><mixed-citation publication-type="journal" xlink:type="simple"><name name-style="western"><surname>Hammarfelt</surname> <given-names>B</given-names></name>, <name name-style="western"><surname>de Rijcke</surname> <given-names>S</given-names></name>. <article-title>Accountability in context: effects of research evaluation systems on publication practices, disciplinary norms, and individual working routines in the faculty of Arts at Uppsala University.</article-title> <source>Research Evaluation</source>. <year>2014</year>;<volume>24</volume>(<issue>1</issue>):<fpage>63</fpage>–<lpage>77</lpage>.</mixed-citation></ref>
</ref-list>
</back>
<sub-article article-type="editor-report" id="pone.0307814.r001" specific-use="decision-letter">
<front-stub>
<article-id pub-id-type="doi">10.1371/journal.pone.0307814.r001</article-id>
<title-group>
<article-title>Decision Letter 0</article-title>
</title-group>
<contrib-group>
<contrib contrib-type="author">
<name name-style="western">
<surname>Cortes</surname>
<given-names>Julian D.</given-names>
</name>
<role>Academic Editor</role>
</contrib>
</contrib-group>
<permissions>
<copyright-year>2024</copyright-year>
<copyright-holder>Julian D. Cortes</copyright-holder>
<license xlink:href="http://creativecommons.org/licenses/by/4.0/">
<license-p>This is an open access article distributed under the terms of the <ext-link ext-link-type="uri" xlink:href="http://creativecommons.org/licenses/by/4.0/" xlink:type="simple">Creative Commons Attribution License</ext-link>, which permits unrestricted use, distribution, and reproduction in any medium, provided the original author and source are credited.</license-p>
</license>
</permissions>
<related-object document-id="10.1371/journal.pone.0307814" document-id-type="doi" document-type="article" id="rel-obj001" link-type="peer-reviewed-article"/>
<custom-meta-group>
<custom-meta>
<meta-name>Submission Version</meta-name>
<meta-value>0</meta-value>
</custom-meta>
</custom-meta-group>
</front-stub>
<body>
<p>
<named-content content-type="letter-date">6 Sep 2023</named-content>
</p>
<p><!-- <div> -->PONE-D-23-25545<!-- </div> --><!-- <div> -->Title: Relational responsibilities: Researchers perspective on current and progressive assessment criteria: a focus group study<!-- </div> --><!-- <div> -->PLOS ONE</p>
<p>Dear Dr. Tijdink,</p>
<p>Thank you for submitting your manuscript to PLOS ONE. After careful consideration, we feel that it has merit but does not fully meet PLOS ONE’s publication criteria as it currently stands. Therefore, we invite you to submit a revised version of the manuscript that addresses the points raised during the review process.</p>
<p>==============================<!-- <div> -->Dear author/s, thanks for submitting your work to PLoS ONE<!-- <span style="white-space:pre"> --> <!-- </span> -->, <!-- </div> --><!-- <div> --> <!-- </div> --><!-- <div> -->I contrasted the core sections of your work with our seven criteria for publication and I consider you could work on the following points before sending it for review: <!-- </div> --><!-- <div> --> <!-- </div> --><!-- <div> -->The study presents the results of original research<!-- </div> --><!-- <div> --> <!-- </div> --><!-- <div> -->Consider enrich the introduction/review with additional literature/evidence on the topic. This, in order to grownd the results and their discussion. For instance, authors could assess and potentially include the following studies to their literature: <!-- </div> --><!-- <div> -->Fochler, M., Felt, U., &amp; Müller, R. (2016). Unsustainable growth, hyper-competition, and worth in life science research: Narrowing evaluative repertoires in doctoral and postdoctoral scientists’ work and lives. Minerva, 54(2), 175–200. <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.1007/s11024-016-9292-y" xlink:type="simple">https://doi.org/10.1007/s11024-016-9292-y</ext-link><!-- </div> --><!-- <div> --> <!-- </div> --><!-- <div> -->Robinson-Garcia, N., Costas, R., Nane, G. F., &amp; van Leeuwen, T. N. (2023). Valuation regimes in academia: Researchers’ attitudes towards their diversity of activities and academic performance. Research Evaluation. <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.1093/RESEVAL/RVAC049 " xlink:type="simple">https://doi.org/10.1093/RESEVAL/RVAC049 </ext-link><!-- </div> --><!-- <div> --> <!-- </div> --><!-- <div> -->Experiments, statistics, and other analyses are performed to a high technical standard and are described in sufficient detail<!-- </div> --><!-- <div> --> <!-- </div> --><!-- <div> -->Consider expand the details on sub-section “Analysis.” The Technique of “Inductive content analysis” just mentioned, however, if it is the main technique to analyze the transcripts, at least, I assume, there are general steps to follow and explain (e.g., 1. Decide the level of analysis: word, word sense, phrase, sentence, themes; 2. Decide how various concepts to code for: develop a pre-defined or interactive set of categories or concepts. Decide either: A. to allow flexibility to add categories through the coding process, or B. to stick with the pre-defined set of categories; 3. Decide whether to code for existence or frequency of a concept. The decision changes the coding process; 4. Decide on how you will distinguish among concepts; 5. Develop rules for coding your texts; 6. Decide what to do with irrelevant information: should this be ignored (e.g. common English words like “the” and “and”), or used to reexamine the coding scheme in the case that it would add to the outcome of coding?; 7. Code the text [<ext-link ext-link-type="uri" xlink:href="https://www.publichealth.columbia.edu/research/population-health-methods/content-analysis" xlink:type="simple">https://www.publichealth.columbia.edu/research/population-health-methods/content-analysis</ext-link>]). <!-- </div> --><!-- <div> -->The whole section on the description and analysis of Table 1, should be part of the “Method” section, not “Results.” <!-- </div> --><!-- <div> --> <!-- </div> --><!-- <div> -->The article adheres to appropriate reporting guidelines and community standards for data availability<!-- </div> --><!-- <div> --> <!-- </div> --><!-- <div> -->Consider expand on the reliability and validity of the methods and results got by using content analysis (<ext-link ext-link-type="uri" xlink:href="https://www.publichealth.columbia.edu/research/population-health-methods/content-analysis" xlink:type="simple">https://www.publichealth.columbia.edu/research/population-health-methods/content-analysis</ext-link>): <!-- </div> --><!-- <div> -->-<!-- <span style="white-space:pre"> --> <!-- </span> -->Reliability (Stability: the tendency for coders to consistently re-code the same data in the same way over a period of time; Reproducibility: tendency for a group of coders to classify categories membership in the same way; Accuracy: extent to which the classification of text corresponds to a standard or norm statistically), and <!-- </div> --><!-- <div> -->-<!-- <span style="white-space:pre"> --> <!-- </span> -->Validity (Closeness of categories: this can be achieved by utilizing multiple classifiers to arrive at an agreed upon definition of each specific category. Using multiple classifiers, a concept category that may be an explicit variable can be broadened to include synonyms or implicit variables; Conclusions: What level of implication is allowable? Do conclusions correctly follow the data? Are results explainable by other phenomena? This becomes especially problematic when using computer software for analysis and distinguishing between synonyms. For example, the word “mine,” variously denotes a personal pronoun, an explosive device, and a deep hole in the ground from which ore is extracted. Software can obtain an accurate count of that word’s occurrence and frequency, but not be able to produce an accurate accounting of the meaning inherent in each particular usage. This problem could throw off one’s results and make any conclusion invalid; Generalizability of the results to a theory: dependent on the clear definitions of concept categories, how they are determined and how reliable they are at measuring the idea one is seeking to measure. Generalizability parallels reliability as much of it depends on the three criteria for reliability.)<!-- </div> --><!-- <div> --> <!-- </div> --><!-- <div> -->The article is presented in an intelligible fashion and is written in standard English<!-- </div> --><!-- <div> --> <!-- </div> --><!-- <div> -->Some readers might not understand colloquialism used in the manuscript, such as “We thus should not throw the baby out with the bath water.” Try to avoid them. <!-- </div> --><!-- <div> --> <!-- </div> --><!-- <div> -->I hope you can incorporate the above suggestions to improve your already valuable work before sending it to review. <!-- </div> --><!-- <div> --> <!-- </div> --><!-- <div> -->Sincerely, <!-- </div> --><!-- <div> --> <!-- </div> --><!-- <div> -->Julián D. Cortés <!-- </div> --><!-- <div> -->Associate Editor<!-- </div> -->==============================</p>
<p>Please submit your revised manuscript by Oct 21 2023 11:59PM. If you will need more time than this to complete your revisions, please reply to this message or contact the journal office at <email xlink:type="simple">plosone@plos.org</email>. When you're ready to submit your revision, log on to <ext-link ext-link-type="uri" xlink:href="https://www.editorialmanager.com/pone/" xlink:type="simple">https://www.editorialmanager.com/pone/</ext-link> and select the 'Submissions Needing Revision' folder to locate your manuscript file.</p>
<p>Please include the following items when submitting your revised manuscript:<!-- </div> --><list list-type="bullet"><list-item><p>A rebuttal letter that responds to each point raised by the academic editor and reviewer(s). You should upload this letter as a separate file labeled 'Response to Reviewers'.</p></list-item><list-item><p>A marked-up copy of your manuscript that highlights changes made to the original version. You should upload this as a separate file labeled 'Revised Manuscript with Track Changes'.</p></list-item><list-item><p>An unmarked version of your revised paper without tracked changes. You should upload this as a separate file labeled 'Manuscript'.</p></list-item></list></p>
<p>If you would like to make changes to your financial disclosure, please include your updated statement in your cover letter. Guidelines for resubmitting your figure files are available below the reviewer comments at the end of this letter.</p>
<p>If applicable, we recommend that you deposit your laboratory protocols in protocols.io to enhance the reproducibility of your results. Protocols.io assigns your protocol its own identifier (DOI) so that it can be cited independently in the future. For instructions see: <ext-link ext-link-type="uri" xlink:href="https://journals.plos.org/plosone/s/submission-guidelines#loc-laboratory-protocols" xlink:type="simple">https://journals.plos.org/plosone/s/submission-guidelines#loc-laboratory-protocols</ext-link>. Additionally, PLOS ONE offers an option for publishing peer-reviewed Lab Protocol articles, which describe protocols hosted on protocols.io. Read more information on sharing protocols at <ext-link ext-link-type="uri" xlink:href="https://plos.org/protocols?utm_medium=editorial-email&amp;utm_source=authorletters&amp;utm_campaign=protocols" xlink:type="simple">https://plos.org/protocols?utm_medium=editorial-email&amp;utm_source=authorletters&amp;utm_campaign=protocols</ext-link>.</p>
<p>We look forward to receiving your revised manuscript.</p>
<p>Kind regards,</p>
<p>Julian D. Cortes</p>
<p>Academic Editor</p>
<p>PLOS ONE</p>
<p>Journal Requirements:</p>
<p>When submitting your revision, we need you to address these additional requirements.</p>
<p>1. Please ensure that your manuscript meets PLOS ONE's style requirements, including those for file naming. The PLOS ONE style templates can be found at </p>
<p><ext-link ext-link-type="uri" xlink:href="https://journals.plos.org/plosone/s/file?id=wjVg/PLOSOne_formatting_sample_main_body.pdf" xlink:type="simple">https://journals.plos.org/plosone/s/file?id=wjVg/PLOSOne_formatting_sample_main_body.pdf</ext-link> and </p>
<p><ext-link ext-link-type="uri" xlink:href="https://journals.plos.org/plosone/s/file?id=ba62/PLOSOne_formatting_sample_title_authors_affiliations.pdf" xlink:type="simple">https://journals.plos.org/plosone/s/file?id=ba62/PLOSOne_formatting_sample_title_authors_affiliations.pdf</ext-link></p>
<p>2. We note that the grant information you provided in the ‘Funding Information’ and ‘Financial Disclosure’ sections do not match. </p>
<p>When you resubmit, please ensure that you provide the correct grant numbers for the awards you received for your study in the ‘Funding Information’ section.</p>
<p>3. Please note that in order to use the direct billing option the corresponding author must be affiliated with the chosen institute. Please either amend your manuscript to change the affiliation or corresponding author, or email us at <email xlink:type="simple">plosone@plos.org</email> with a request to remove this option.</p>
<p>4. In your Data Availability statement, you have not specified where the minimal data set underlying the results described in your manuscript can be found. PLOS defines a study's minimal data set as the underlying data used to reach the conclusions drawn in the manuscript and any additional data required to replicate the reported study findings in their entirety. All PLOS journals require that the minimal data set be made fully available. For more information about our data policy, please see <ext-link ext-link-type="uri" xlink:href="http://journals.plos.org/plosone/s/data-availability" xlink:type="simple">http://journals.plos.org/plosone/s/data-availability</ext-link>.</p>
<p>"Upon re-submitting your revised manuscript, please upload your study’s minimal underlying data set as either Supporting Information files or to a stable, public repository and include the relevant URLs, DOIs, or accession numbers within your revised cover letter. For a list of acceptable repositories, please see <ext-link ext-link-type="uri" xlink:href="http://journals.plos.org/plosone/s/data-availability#loc-recommended-repositories" xlink:type="simple">http://journals.plos.org/plosone/s/data-availability#loc-recommended-repositories</ext-link>. Any potentially identifying patient information must be fully anonymized.</p>
<p>Important: If there are ethical or legal restrictions to sharing your data publicly, please explain these restrictions in detail. Please see our guidelines for more information on what we consider unacceptable restrictions to publicly sharing data: <ext-link ext-link-type="uri" xlink:href="http://journals.plos.org/plosone/s/data-availability#loc-unacceptable-data-access-restrictions" xlink:type="simple">http://journals.plos.org/plosone/s/data-availability#loc-unacceptable-data-access-restrictions</ext-link>. Note that it is not acceptable for the authors to be the sole named individuals responsible for ensuring data access.</p>
<p>We will update your Data Availability statement to reflect the information you provide in your cover letter.</p>
<p>5. Please include a copy of Table 3 which you refer to in your text on page 8.</p>
<p>6. Please include captions for your Supporting Information files at the end of your manuscript, and update any in-text citations to match accordingly. Please see our Supporting Information guidelines for more information: <ext-link ext-link-type="uri" xlink:href="http://journals.plos.org/plosone/s/supporting-information. " xlink:type="simple">http://journals.plos.org/plosone/s/supporting-information. </ext-link></p>
<p>Additional Editor Comments:</p>
<p>Dear author/s, thanks for submitting your work to PLoS ONE ,</p>
<p>I contrasted the core sections of your work with our seven criteria for publication and I consider you could work on the following points before sending it for review:</p>
<p>The study presents the results of original research</p>
<p>Consider enrich the introduction/review with additional literature/evidence on the topic. This, in order to grownd the results and their discussion. For instance, authors could assess and potentially include the following studies to their literature:</p>
<p>Fochler, M., Felt, U., &amp; Müller, R. (2016). Unsustainable growth, hyper-competition, and worth in life science research: Narrowing evaluative repertoires in doctoral and postdoctoral scientists’ work and lives. Minerva, 54(2), 175–200. <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.1007/s11024-016-9292-y" xlink:type="simple">https://doi.org/10.1007/s11024-016-9292-y</ext-link></p>
<p>Robinson-Garcia, N., Costas, R., Nane, G. F., &amp; van Leeuwen, T. N. (2023). Valuation regimes in academia: Researchers’ attitudes towards their diversity of activities and academic performance. Research Evaluation. <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.1093/RESEVAL/RVAC049" xlink:type="simple">https://doi.org/10.1093/RESEVAL/RVAC049</ext-link></p>
<p>Experiments, statistics, and other analyses are performed to a high technical standard and are described in sufficient detail</p>
<p>Consider expand the details on sub-section “Analysis.” The Technique of “Inductive content analysis” just mentioned, however, if it is the main technique to analyze the transcripts, at least, I assume, there are general steps to follow and explain (e.g., 1. Decide the level of analysis: word, word sense, phrase, sentence, themes; 2. Decide how various concepts to code for: develop a pre-defined or interactive set of categories or concepts. Decide either: A. to allow flexibility to add categories through the coding process, or B. to stick with the pre-defined set of categories; 3. Decide whether to code for existence or frequency of a concept. The decision changes the coding process; 4. Decide on how you will distinguish among concepts; 5. Develop rules for coding your texts; 6. Decide what to do with irrelevant information: should this be ignored (e.g. common English words like “the” and “and”), or used to reexamine the coding scheme in the case that it would add to the outcome of coding?; 7. Code the text [<ext-link ext-link-type="uri" xlink:href="https://www.publichealth.columbia.edu/research/population-health-methods/content-analysis" xlink:type="simple">https://www.publichealth.columbia.edu/research/population-health-methods/content-analysis</ext-link>]).</p>
<p>The whole section on the description and analysis of Table 1, should be part of the “Method” section, not “Results.”</p>
<p>The article adheres to appropriate reporting guidelines and community standards for data availability</p>
<p>Consider expand on the reliability and validity of the methods and results got by using content analysis (<ext-link ext-link-type="uri" xlink:href="https://www.publichealth.columbia.edu/research/population-health-methods/content-analysis" xlink:type="simple">https://www.publichealth.columbia.edu/research/population-health-methods/content-analysis</ext-link>):</p>
<p>- Reliability (Stability: the tendency for coders to consistently re-code the same data in the same way over a period of time; Reproducibility: tendency for a group of coders to classify categories membership in the same way; Accuracy: extent to which the classification of text corresponds to a standard or norm statistically), and</p>
<p>- Validity (Closeness of categories: this can be achieved by utilizing multiple classifiers to arrive at an agreed upon definition of each specific category. Using multiple classifiers, a concept category that may be an explicit variable can be broadened to include synonyms or implicit variables; Conclusions: What level of implication is allowable? Do conclusions correctly follow the data? Are results explainable by other phenomena? This becomes especially problematic when using computer software for analysis and distinguishing between synonyms. For example, the word “mine,” variously denotes a personal pronoun, an explosive device, and a deep hole in the ground from which ore is extracted. Software can obtain an accurate count of that word’s occurrence and frequency, but not be able to produce an accurate accounting of the meaning inherent in each particular usage. This problem could throw off one’s results and make any conclusion invalid; Generalizability of the results to a theory: dependent on the clear definitions of concept categories, how they are determined and how reliable they are at measuring the idea one is seeking to measure. Generalizability parallels reliability as much of it depends on the three criteria for reliability.)</p>
<p>The article is presented in an intelligible fashion and is written in standard English</p>
<p>Some readers might not understand colloquialism used in the manuscript, such as “We thus should not throw the baby out with the bath water.” Try to avoid them.</p>
<p>I hope you can incorporate the above suggestions to improve your already valuable work before sending it to review.</p>
<p>Sincerely,</p>
<p>Julián D. Cortés</p>
<p>Associate Editor</p>
<p>[Note: HTML markup is below. Please do not edit.]</p>
<p>[NOTE: If reviewer comments were submitted as an attachment file, they will be attached to this email and accessible via the submission site. Please log into your account, locate the manuscript record, and check for the action link "View Attachments". If this link does not appear, there are no attachment files.]</p>
<p>While revising your submission, please upload your figure files to the Preflight Analysis and Conversion Engine (PACE) digital diagnostic tool, <ext-link ext-link-type="uri" xlink:href="https://pacev2.apexcovantage.com/" xlink:type="simple">https://pacev2.apexcovantage.com/</ext-link>. PACE helps ensure that figures meet PLOS requirements. To use PACE, you must first register as a user. Registration is free. Then, login and navigate to the UPLOAD tab, where you will find detailed instructions on how to use the tool. If you encounter any issues or have any questions when using PACE, please email PLOS at <email xlink:type="simple">figures@plos.org</email>. Please note that Supporting Information files do not need this step.</p>
</body>
</sub-article>
<sub-article article-type="author-comment" id="pone.0307814.r002">
<front-stub>
<article-id pub-id-type="doi">10.1371/journal.pone.0307814.r002</article-id>
<title-group>
<article-title>Author response to Decision Letter 0</article-title>
</title-group>
<related-object document-id="10.1371/journal.pone.0307814" document-id-type="doi" document-type="peer-reviewed-article" id="rel-obj002" link-type="rebutted-decision-letter" object-id="10.1371/journal.pone.0307814.r001" object-id-type="doi" object-type="decision-letter"/>
<custom-meta-group>
<custom-meta>
<meta-name>Submission Version</meta-name>
<meta-value>1</meta-value>
</custom-meta>
</custom-meta-group>
</front-stub>
<body>
<p>
<named-content content-type="author-response-date">22 Nov 2023</named-content>
</p>
<p>Dear Prof. Cortés,</p>
<p>Thank you for your review of our article, recently submitted to PLoS One. We have read it with interest and below you will find answers in italics to the points you have raised in a point by point fashion. If we have changed the text, we indicate this and write this text in blue. This letter is also included in the attached files.</p>
<p>Thanks again for your valuable feedback. We look forward to hear from you.</p>
<p>Warm regards, also on behalf of the authors.</p>
<p>Joeri Tijdink, MD PhD</p>
<p>Amsterdam</p>
<p>Editor: I contrasted the core sections of your work with our seven criteria for publication and I consider you could work on the following points before sending it for review: </p>
<p>1. The study presents the results of original research</p>
<p>Consider enrich the introduction/review with additional literature/evidence on the topic. This, in order to grownd the results and their discussion. For instance, authors could assess and potentially include the following studies to their literature: </p>
<p>Fochler, M., Felt, U., &amp; Müller, R. (2016). Unsustainable growth, hyper-competition, and worth in life science research: Narrowing evaluative repertoires in doctoral and postdoctoral scientists’ work and lives. Minerva, 54(2), 175–200. <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.1007/s11024-016-9292-y" xlink:type="simple">https://doi.org/10.1007/s11024-016-9292-y</ext-link></p>
<p>Robinson-Garcia, N., Costas, R., Nane, G. F., &amp; van Leeuwen, T. N. (2023). Valuation regimes in academia: Researchers’ attitudes towards their diversity of activities and academic performance. Research Evaluation. <ext-link ext-link-type="uri" xlink:href="https://doi.org/10.1093/RESEVAL/RVAC049" xlink:type="simple">https://doi.org/10.1093/RESEVAL/RVAC049</ext-link> </p>
<p>Answer: Thank you for your thoughtful suggestion. We are happy you brought up these references as they have inspired us in the debate on evaluation and academic performance. We have expanded the introduction section with the two references and have added the following text to the introduction. We now write:</p>
<p>“One study alluded to this by emphasizing researchers perceptions of valuation regimes beyond traditional research activities and how current systems and criteria can influence behavior and choices regarding the activities they prioritize (14,15). This is also influenced by intense competition can narrow the range of activities and goals that scientists pursue due to the limited set of criteria and values by which scientists are judged and that reevaluating the criteria in academia is necessary to create a healthier and more sustainable research culture (15)”. </p>
<p>2. Experiments, statistics, and other analyses are performed to a high technical standard and are described in sufficient detail</p>
<p>Consider expand the details on sub-section “Analysis.” The Technique of “Inductive content analysis” just mentioned, however, if it is the main technique to analyze the transcripts, at least, I assume, there are general steps to follow and explain (e.g., 1. Decide the level of analysis: word, word sense, phrase, sentence, themes; 2. Decide how various concepts to code for: develop a pre-defined or interactive set of categories or concepts. Decide either: A. to allow flexibility to add categories through the coding process, or B. to stick with the pre-defined set of categories; 3. Decide whether to code for existence or frequency of a concept. The decision changes the coding process; 4. Decide on how you will distinguish among concepts; 5. Develop rules for coding your texts; 6. Decide what to do with irrelevant information: should this be ignored (e.g. common English words like “the” and “and”), or used to reexamine the coding scheme in the case that it would add to the outcome of coding?; 7. Code the text [<ext-link ext-link-type="uri" xlink:href="https://www.publichealth.columbia.edu/research/population-health-methods/content-analysis" xlink:type="simple">https://www.publichealth.columbia.edu/research/population-health-methods/content-analysis</ext-link>]). </p>
<p>The whole section on the description and analysis of Table 1, should be part of the “Method” section, not “Results.” </p>
<p>Answer: Thanks for this. We have extended our description of the inductive content analysis in the manuscript. Furthermore, we have moved the ‘descriptive information’ including table 1 to the methods section. We now write in the methods section under analysis on page 7:</p>
<p>“We used inductive content analysis for the analysis of the transcripts of the focus groups and took the following steps in this process. First we coded the data on a phrase level. Next, we established a tentative set of themes to analyze the selected phrases, while allowing for flexibility to revise and update these themes during the process, if necessary. Then we analyzed the different themes and summarized these themes. This process of inductive content analysis helps us to understand complex discussions and bring them back to meaningful themes (16)”.</p>
<p>3. The article adheres to appropriate reporting guidelines and community standards for data availability</p>
<p>Consider expand on the reliability and validity of the methods and results got by using content analysis (<ext-link ext-link-type="uri" xlink:href="https://www.publichealth.columbia.edu/research/population-health-methods/content-analysis" xlink:type="simple">https://www.publichealth.columbia.edu/research/population-health-methods/content-analysis</ext-link>): </p>
<p>- Reliability (Stability: the tendency for coders to consistently re-code the same data in the same way over a period of time; Reproducibility: tendency for a group of coders to classify categories membership in the same way; Accuracy: extent to which the classification of text corresponds to a standard or norm statistically), and </p>
<p>- Validity (Closeness of categories: this can be achieved by utilizing multiple classifiers to arrive at an agreed upon definition of each specific category. Using multiple classifiers, a concept category that may be an explicit variable can be broadened to include synonyms or implicit variables; Conclusions: What level of implication is allowable? Do conclusions correctly follow the data? Are results explainable by other phenomena? This becomes especially problematic when using computer software for analysis and distinguishing between synonyms. For example, the word “mine,” variously denotes a personal pronoun, an explosive device, and a deep hole in the ground from which ore is extracted. Software can obtain an accurate count of that word’s occurrence and frequency, but not be able to produce an accurate accounting of the meaning inherent in each particular usage. This problem could throw off one’s results and make any conclusion invalid; Generalizability of the results to a theory: dependent on the clear definitions of concept categories, how they are determined and how reliable they are at measuring the idea one is seeking to measure. Generalizability parallels reliability as much of it depends on the three criteria for reliability.)</p>
<p>Answers: Thank you. Personally, I like this a lot to include this in an article as it tells us more about the quality and reproducibility of our work. We have added these considerations to the discussion section in the interpretation of the data. We have added the following text:</p>
<p>“Third, it is good to reflect on both reliability and validity of our findings. Considering the reproducibility of our work, two persons from our research team coded and analyzed the transcripts to assure that we found and classified similar categories and themes. Considering the validity, JT and GD have discussed the content of the themes in more detail on several occasions to assure that our categorization is similar and explicit. Moreover, we have discussed generalizability and reflect on this in the limitation section of this article.”</p>
<p>4. The article is presented in an intelligible fashion and is written in standard English</p>
<p>Some readers might not understand colloquialism used in the manuscript, such as “We thus should not throw the baby out with the bath water.” Try to avoid them. </p>
<p>Answer: Thank you. We have changed this sentence and we now write in the discussion section: </p>
<p>“We thus should exercise caution to avoid discarding valuable elements or aspects when making comprehensive changes or judgments, as prematurely eliminating the essential along with the dispensable may lead to unintended consequences.”</p>
</body>
</sub-article>
<sub-article article-type="aggregated-review-documents" id="pone.0307814.r003" specific-use="decision-letter">
<front-stub>
<article-id pub-id-type="doi">10.1371/journal.pone.0307814.r003</article-id>
<title-group>
<article-title>Decision Letter 1</article-title>
</title-group>
<contrib-group>
<contrib contrib-type="author">
<name name-style="western">
<surname>Cortes</surname>
<given-names>Julian D.</given-names>
</name>
<role>Academic Editor</role>
</contrib>
</contrib-group>
<permissions>
<copyright-year>2024</copyright-year>
<copyright-holder>Julian D. Cortes</copyright-holder>
<license xlink:href="http://creativecommons.org/licenses/by/4.0/">
<license-p>This is an open access article distributed under the terms of the <ext-link ext-link-type="uri" xlink:href="http://creativecommons.org/licenses/by/4.0/" xlink:type="simple">Creative Commons Attribution License</ext-link>, which permits unrestricted use, distribution, and reproduction in any medium, provided the original author and source are credited.</license-p>
</license>
</permissions>
<related-object document-id="10.1371/journal.pone.0307814" document-id-type="doi" document-type="article" id="rel-obj003" link-type="peer-reviewed-article"/>
<custom-meta-group>
<custom-meta>
<meta-name>Submission Version</meta-name>
<meta-value>1</meta-value>
</custom-meta>
</custom-meta-group>
</front-stub>
<body>
<p>
<named-content content-type="letter-date">13 Feb 2024</named-content>
</p>
<p><!-- <div> -->PONE-D-23-25545R1<!-- </div> --><!-- <div> -->Title: Relational responsibilities: Researchers perspective on current and progressive assessment criteria: a focus group study<!-- </div> --><!-- <div> -->PLOS ONE</p>
<p>Dear Dr. Tijdink,</p>
<p>Thank you for submitting your manuscript to PLOS ONE. After careful consideration, we feel that it has merit but does not fully meet PLOS ONE’s publication criteria as it currently stands. Therefore, we invite you to submit a revised version of the manuscript that addresses the points raised during the review process.</p>
<p>==============================</p>
<p>Dear authors,</p>
<p>In view of the referees’ feedback and my own reading of your paper, we invite you to address all issues noted below, most of which are relatively minor in nature, but nonetheless essential. In particular, a few adjustments in literature review; counts/quantification of the focus group themes; and more discussion rooted in existing reward model structure.</p>
<p>We look forward to your revised version.</p>
<p>Sincerely,</p>
<p>Julián D. Cortés<!-- </div> --><!-- <div> -->==============================</p>
<p>Please submit your revised manuscript by Mar 29 2024 11:59PM. If you will need more time than this to complete your revisions, please reply to this message or contact the journal office at <email xlink:type="simple">plosone@plos.org</email>. When you're ready to submit your revision, log on to <ext-link ext-link-type="uri" xlink:href="https://www.editorialmanager.com/pone/" xlink:type="simple">https://www.editorialmanager.com/pone/</ext-link> and select the 'Submissions Needing Revision' folder to locate your manuscript file.</p>
<p>Please include the following items when submitting your revised manuscript:<!-- </div> --><list list-type="bullet"><list-item><p>A rebuttal letter that responds to each point raised by the academic editor and reviewer(s). You should upload this letter as a separate file labeled 'Response to Reviewers'.</p></list-item><list-item><p>A marked-up copy of your manuscript that highlights changes made to the original version. You should upload this as a separate file labeled 'Revised Manuscript with Track Changes'.</p></list-item><list-item><p>An unmarked version of your revised paper without tracked changes. You should upload this as a separate file labeled 'Manuscript'.</p></list-item></list><!-- <div> -->If you would like to make changes to your financial disclosure, please include your updated statement in your cover letter. Guidelines for resubmitting your figure files are available below the reviewer comments at the end of this letter.</p>
<p>If applicable, we recommend that you deposit your laboratory protocols in protocols.io to enhance the reproducibility of your results. Protocols.io assigns your protocol its own identifier (DOI) so that it can be cited independently in the future. For instructions see: <ext-link ext-link-type="uri" xlink:href="https://journals.plos.org/plosone/s/submission-guidelines#loc-laboratory-protocols" xlink:type="simple">https://journals.plos.org/plosone/s/submission-guidelines#loc-laboratory-protocols</ext-link>. Additionally, PLOS ONE offers an option for publishing peer-reviewed Lab Protocol articles, which describe protocols hosted on protocols.io. Read more information on sharing protocols at <ext-link ext-link-type="uri" xlink:href="https://plos.org/protocols?utm_medium=editorial-email&amp;utm_source=authorletters&amp;utm_campaign=protocols" xlink:type="simple">https://plos.org/protocols?utm_medium=editorial-email&amp;utm_source=authorletters&amp;utm_campaign=protocols</ext-link>.</p>
<p>We look forward to receiving your revised manuscript.</p>
<p>Kind regards,</p>
<p>Julian D. Cortes</p>
<p>Academic Editor</p>
<p>PLOS ONE</p>
<p>Journal Requirements:</p>
<p>Please review your reference list to ensure that it is complete and correct. If you have cited papers that have been retracted, please include the rationale for doing so in the manuscript text, or remove these references and replace them with relevant current references. Any changes to the reference list should be mentioned in the rebuttal letter that accompanies your revised manuscript. If you need to cite a retracted article, indicate the article’s retracted status in the References list and also include a citation and full reference for the retraction notice.</p>
<p>[Note: HTML markup is below. Please do not edit.]</p>
<p>Reviewers' comments:</p>
<p>Reviewer's Responses to Questions</p>
<p><!-- <font color="black"> --><bold>Comments to the Author</bold></p>
<p>1. If the authors have adequately addressed your comments raised in a previous round of review and you feel that this manuscript is now acceptable for publication, you may indicate that here to bypass the “Comments to the Author” section, enter your conflict of interest statement in the “Confidential to Editor” section, and submit your "Accept" recommendation.<!-- </font> --></p>
<p>Reviewer #1: All comments have been addressed</p>
<p>Reviewer #2: (No Response)</p>
<p>**********</p>
<p><!-- <font color="black"> -->2. Is the manuscript technically sound, and do the data support the conclusions?</p>
<p>The manuscript must describe a technically sound piece of scientific research with data that supports the conclusions. Experiments must have been conducted rigorously, with appropriate controls, replication, and sample sizes. The conclusions must be drawn appropriately based on the data presented. <!-- </font> --></p>
<p>Reviewer #1: Yes</p>
<p>Reviewer #2: Yes</p>
<p>**********</p>
<p><!-- <font color="black"> -->3. Has the statistical analysis been performed appropriately and rigorously? <!-- </font> --></p>
<p>Reviewer #1: Yes</p>
<p>Reviewer #2: N/A</p>
<p>**********</p>
<p><!-- <font color="black"> -->4. Have the authors made all data underlying the findings in their manuscript fully available?</p>
<p>The <ext-link ext-link-type="uri" xlink:href="http://www.plosone.org/static/policies.action#sharing" xlink:type="simple">PLOS Data policy</ext-link> requires authors to make all data underlying the findings described in their manuscript fully available without restriction, with rare exception (please refer to the Data Availability Statement in the manuscript PDF file). The data should be provided as part of the manuscript or its supporting information, or deposited to a public repository. For example, in addition to summary statistics, the data points behind means, medians and variance measures should be available. If there are restrictions on publicly sharing data—e.g. participant privacy or use of data from a third party—those must be specified.<!-- </font> --></p>
<p>Reviewer #1: Yes</p>
<p>Reviewer #2: Yes</p>
<p>**********</p>
<p><!-- <font color="black"> -->5. Is the manuscript presented in an intelligible fashion and written in standard English?</p>
<p>PLOS ONE does not copyedit accepted manuscripts, so the language in submitted articles must be clear, correct, and unambiguous. Any typographical or grammatical errors should be corrected at revision, so please note any specific errors here.<!-- </font> --></p>
<p>Reviewer #1: Yes</p>
<p>Reviewer #2: Yes</p>
<p>**********</p>
<p><!-- <font color="black"> -->6. Review Comments to the Author</p>
<p>Please use the space provided to explain your answers to the questions above. You may also include additional comments for the author, including concerns about dual publication, research ethics, or publication ethics. (Please upload your review as an attachment if it exceeds 20,000 characters)<!-- </font> --></p>
<p>Reviewer #1: Perhaps a table with a summary of results could help to point out the main themes/categories/solutions based on the results</p>
<p>Reviewer #2: It was a pleasure reading this important work with fascinating findings. There are only a few items to address to strengthen the manuscript for publication.</p>
<p>(1) An expanded literature review;</p>
<p>(2) Counts/quantification of the focus group themes</p>
<p>(3) More discussion rooted in existing reward model structure</p>
<p>(4) Limitations of study improved</p>
<p>(1) The literature could include its own section to connect to existing work in this area. Journal of Research</p>
<p>Administration, Chronicle of Higher Ed, and others have research to give more context to scholarly communication, misconduct statistics, team science, and any additional context would help connect this work to the existing body of knowledge related to P&amp;T.</p>
<p>(2) The direct quotes and themes are very good, but having counts of mentions or number of participants that represented/agreed with a theme or comment would give more detail to readers about how prevalent the sentiments were. Perhaps, even a table of themes and examples of each would present this in a more readable manner than the paragraphs of text.</p>
<p>(3) CRediT (Contributor Roles Taxonomy) may be an easy inclusion to explain how roles on teams do get credit now in some fields on large research teams. There are other metrics you can cite to connect the discussion and recommendations to current practices across the hard sciences. The entire discussion section would benefit from the authors adding more ideas and recommendations. What there is great, I just would like to see more.</p>
<p>(4) The limitations of the study are clear, but having a stand alone section of the limitations and ways to address them in future research would increase readability. Towards the end of that section a sentence just starts "Since we" and then is blank... so what should be there?</p>
<p>Overall, more subheadings and more concise writing style would also help.</p>
<p>**********</p>
<p><!-- <font color="black"> -->7. PLOS authors have the option to publish the peer review history of their article (<ext-link ext-link-type="uri" xlink:href="https://journals.plos.org/plosone/s/editorial-and-peer-review-process#loc-peer-review-history" xlink:type="simple">what does this mean?</ext-link>). If published, this will include your full peer review and any attached files.</p>
<p>If you choose “no”, your identity will remain anonymous but your review may still be made public.</p>
<p><bold>Do you want your identity to be public for this peer review?</bold> For information about this choice, including consent withdrawal, please see our <ext-link ext-link-type="uri" xlink:href="https://www.plos.org/privacy-policy" xlink:type="simple">Privacy Policy</ext-link>.<!-- </font> --></p>
<p>Reviewer #1: No</p>
<p>Reviewer #2: <bold>Yes: </bold>Bradley Wade Bishop</p>
<p>**********</p>
<p>[NOTE: If reviewer comments were submitted as an attachment file, they will be attached to this email and accessible via the submission site. Please log into your account, locate the manuscript record, and check for the action link "View Attachments". If this link does not appear, there are no attachment files.]</p>
<p>While revising your submission, please upload your figure files to the Preflight Analysis and Conversion Engine (PACE) digital diagnostic tool, <ext-link ext-link-type="uri" xlink:href="https://pacev2.apexcovantage.com/" xlink:type="simple">https://pacev2.apexcovantage.com/</ext-link>. PACE helps ensure that figures meet PLOS requirements. To use PACE, you must first register as a user. Registration is free. Then, login and navigate to the UPLOAD tab, where you will find detailed instructions on how to use the tool. If you encounter any issues or have any questions when using PACE, please email PLOS at <email xlink:type="simple">figures@plos.org</email>. Please note that Supporting Information files do not need this step.</p>
</body>
</sub-article>
<sub-article article-type="author-comment" id="pone.0307814.r004">
<front-stub>
<article-id pub-id-type="doi">10.1371/journal.pone.0307814.r004</article-id>
<title-group>
<article-title>Author response to Decision Letter 1</article-title>
</title-group>
<related-object document-id="10.1371/journal.pone.0307814" document-id-type="doi" document-type="peer-reviewed-article" id="rel-obj004" link-type="rebutted-decision-letter" object-id="10.1371/journal.pone.0307814.r003" object-id-type="doi" object-type="decision-letter"/>
<custom-meta-group>
<custom-meta>
<meta-name>Submission Version</meta-name>
<meta-value>2</meta-value>
</custom-meta>
</custom-meta-group>
</front-stub>
<body>
<p>
<named-content content-type="author-response-date">27 Jun 2024</named-content>
</p>
<p>See our response in the cover letter</p>
</body>
</sub-article>
<sub-article article-type="editor-report" id="pone.0307814.r005" specific-use="decision-letter">
<front-stub>
<article-id pub-id-type="doi">10.1371/journal.pone.0307814.r005</article-id>
<title-group>
<article-title>Decision Letter 2</article-title>
</title-group>
<contrib-group>
<contrib contrib-type="author">
<name name-style="western">
<surname>Cortes</surname>
<given-names>Julian D.</given-names>
</name>
<role>Academic Editor</role>
</contrib>
</contrib-group>
<permissions>
<copyright-year>2024</copyright-year>
<copyright-holder>Julian D. Cortes</copyright-holder>
<license xlink:href="http://creativecommons.org/licenses/by/4.0/">
<license-p>This is an open access article distributed under the terms of the <ext-link ext-link-type="uri" xlink:href="http://creativecommons.org/licenses/by/4.0/" xlink:type="simple">Creative Commons Attribution License</ext-link>, which permits unrestricted use, distribution, and reproduction in any medium, provided the original author and source are credited.</license-p>
</license>
</permissions>
<related-object document-id="10.1371/journal.pone.0307814" document-id-type="doi" document-type="article" id="rel-obj005" link-type="peer-reviewed-article"/>
<custom-meta-group>
<custom-meta>
<meta-name>Submission Version</meta-name>
<meta-value>2</meta-value>
</custom-meta>
</custom-meta-group>
</front-stub>
<body>
<p>
<named-content content-type="letter-date">12 Jul 2024</named-content>
</p>
<p>Title: Relational responsibilities: Researchers perspective on current and progressive assessment criteria: a focus group study</p>
<p>PONE-D-23-25545R2</p>
<p>Dear Dr. Tijdink,</p>
<p>We’re pleased to inform you that your manuscript has been judged scientifically suitable for publication and will be formally accepted for publication once it meets all outstanding technical requirements.</p>
<p>Within one week, you’ll receive an e-mail detailing the required amendments. When these have been addressed, you’ll receive a formal acceptance letter and your manuscript will be scheduled for publication.</p>
<p>An invoice will be generated when your article is formally accepted. Please note, if your institution has a publishing partnership with PLOS and your article meets the relevant criteria, all or part of your publication costs will be covered. Please make sure your user information is up-to-date by logging into Editorial Manager at <ext-link ext-link-type="uri" xlink:href="https://www.editorialmanager.com/pone/" xlink:type="simple">Editorial Manager®</ext-link> and clicking the ‘Update My Information' link at the top of the page. If you have any questions relating to publication charges, please contact our Author Billing department directly at <email xlink:type="simple">authorbilling@plos.org</email>.</p>
<p>If your institution or institutions have a press office, please notify them about your upcoming paper to help maximize its impact. If they’ll be preparing press materials, please inform our press team as soon as possible -- no later than 48 hours after receiving the formal acceptance. Your manuscript will remain under strict press embargo until 2 pm Eastern Time on the date of publication. For more information, please contact <email xlink:type="simple">onepress@plos.org</email>.</p>
<p>Kind regards,</p>
<p>Julian D. Cortes</p>
<p>Academic Editor</p>
<p>PLOS ONE</p>
<p>Additional Editor Comments (optional):</p>
<p>Reviewers' comments:</p>
</body>
</sub-article>
<sub-article article-type="editor-report" id="pone.0307814.r006" specific-use="acceptance-letter">
<front-stub>
<article-id pub-id-type="doi">10.1371/journal.pone.0307814.r006</article-id>
<title-group>
<article-title>Acceptance letter</article-title>
</title-group>
<contrib-group>
<contrib contrib-type="author">
<name name-style="western">
<surname>Cortes</surname>
<given-names>Julian D.</given-names>
</name>
<role>Academic Editor</role>
</contrib>
</contrib-group>
<permissions>
<copyright-year>2024</copyright-year>
<copyright-holder>Julian D. Cortes</copyright-holder>
<license xlink:href="http://creativecommons.org/licenses/by/4.0/">
<license-p>This is an open access article distributed under the terms of the <ext-link ext-link-type="uri" xlink:href="http://creativecommons.org/licenses/by/4.0/" xlink:type="simple">Creative Commons Attribution License</ext-link>, which permits unrestricted use, distribution, and reproduction in any medium, provided the original author and source are credited.</license-p>
</license>
</permissions>
<related-object document-id="10.1371/journal.pone.0307814" document-id-type="doi" document-type="article" id="rel-obj006" link-type="peer-reviewed-article"/>
</front-stub>
<body>
<p>
<named-content content-type="letter-date">24 Jul 2024</named-content>
</p>
<p>PONE-D-23-25545R2 </p>
<p>PLOS ONE</p>
<p>Dear Dr.  Tijdink, </p>
<p>I'm pleased to inform you that your manuscript has been deemed suitable for publication in PLOS ONE. Congratulations! Your manuscript is now being handed over to our production team.</p>
<p>At this stage, our production department will prepare your paper for publication. This includes ensuring the following:</p>
<p>* All references, tables, and figures are properly cited</p>
<p>* All relevant supporting information is included in the manuscript submission,</p>
<p>* There are no issues that prevent the paper from being properly typeset</p>
<p>If revisions are needed, the production department will contact you directly to resolve them. If no revisions are needed, you will receive an email when the publication date has been set. At this time, we do not offer pre-publication proofs to authors during production of the accepted work. Please keep in mind that we are working through a large volume of accepted articles, so please give us a few weeks to review your paper and let you know the next and final steps. </p>
<p>Lastly, if your institution or institutions have a press office, please let them know about your upcoming paper now to help maximize its impact. If they'll be preparing press materials, please inform our press team within the next 48 hours. Your manuscript will remain under strict press embargo until 2 pm Eastern Time on the date of publication. For more information, please contact <email xlink:type="simple">onepress@plos.org</email>.</p>
<p>If we can help with anything else, please email us at <email xlink:type="simple">customercare@plos.org</email>.</p>
<p>Thank you for submitting your work to PLOS ONE and supporting open access. </p>
<p>Kind regards, </p>
<p>PLOS ONE Editorial Office Staff</p>
<p>on behalf of</p>
<p>Professor Julian D. Cortes </p>
<p>Academic Editor</p>
<p>PLOS ONE</p>
</body>
</sub-article>
</article>